def test_get_from_job(self): """Test getting from job.""" data_types.Job(name='job', environment_string=('ISSUE_TRACKER = from_job\n' 'HELP_URL = help_url\n')).put() self.assertEqual('from_job', data_handler.get_issue_tracker_name('job'))
def get(self): """Handles get request.""" email = helpers.get_user_email() if not email: raise helpers.AccessDeniedException() is_privileged_or_domain_user = access.has_access( need_privileged_access=False) if is_privileged_or_domain_user or _is_uploader_allowed(email): # Privileged, domain and upload users can see all job and fuzzer names. allowed_jobs = data_handler.get_all_job_type_names() allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children( include_parents=True) else: # Check if this is an external user with access to certain fuzzers/jobs. allowed_jobs = external_users.allowed_jobs_for_user(email) allowed_fuzzers = external_users.allowed_fuzzers_for_user( email, include_from_jobs=True) if not allowed_fuzzers and not allowed_jobs: raise helpers.AccessDeniedException() has_issue_tracker = bool(data_handler.get_issue_tracker_name()) result, params = get_result() return self.render( 'upload.html', { 'fieldValues': { 'blackboxFuzzers': filter_blackbox_fuzzers(allowed_fuzzers), 'jobs': allowed_jobs, 'libfuzzerTargets': filter_target_names(allowed_fuzzers, 'libFuzzer'), 'aflTargets': filter_target_names(allowed_fuzzers, 'afl'), 'honggfuzzTargets': filter_target_names(allowed_fuzzers, 'honggfuzz'), 'isChromium': utils.is_chromium(), 'sandboxedJobs': data_types.INTERNAL_SANDBOXED_JOB_TYPES, 'csrfToken': form.generate_csrf_token(), 'isExternalUser': not is_privileged_or_domain_user, 'uploadInfo': gcs.prepare_blob_upload()._asdict(), 'hasIssueTracker': has_issue_tracker, }, 'params': params, 'result': result })
def get(self): """Handles get request.""" email = helpers.get_user_email() if not email: raise helpers.AccessDeniedException() is_privileged_or_domain_user = access.has_access( need_privileged_access=False) if is_privileged_or_domain_user or _is_uploader_allowed(email): # Privileged, domain and upload users can see all job and fuzzer names. allowed_jobs = data_handler.get_all_job_type_names() allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children( include_parents=True) else: # Check if this is an external user with access to certain fuzzers/jobs. allowed_jobs = external_users.allowed_jobs_for_user(email) allowed_fuzzers = external_users.allowed_fuzzers_for_user( email, include_from_jobs=True) if not allowed_fuzzers and not allowed_jobs: raise helpers.AccessDeniedException() has_issue_tracker = bool(data_handler.get_issue_tracker_name()) result, params = get_result(self) self.render( "upload.html", { "fieldValues": { "jobs": allowed_jobs, "libfuzzerTargets": filter_target_names(allowed_fuzzers, "libFuzzer"), "aflTargets": filter_target_names(allowed_fuzzers, "afl"), "isChromium": utils.is_chromium(), "sandboxedJobs": data_types.INTERNAL_SANDBOXED_JOB_TYPES, "csrfToken": form.generate_csrf_token(), "isExternalUser": not is_privileged_or_domain_user, "uploadInfo": gcs.prepare_blob_upload()._asdict(), "hasIssueTracker": has_issue_tracker, }, "params": params, "result": result, }, )
def get_issue_tracker(tracker_type, project_name=None, use_cache=False): """Get the issue tracker with the given type and name.""" # TODO(ochang): Actually use `tracker_type`. assert tracker_type == 'monorail' if not project_name: from datastore import data_handler project_name = data_handler.get_issue_tracker_name() itm = _get_issue_tracker_manager_for_project(project_name, use_cache=use_cache) if itm is None: return None return monorail.IssueTracker(itm)
def get_issue_tracker(project_name=None): """Get the issue tracker with the given type and name.""" issue_tracker_config = local_config.IssueTrackerConfig() if not project_name: from datastore import data_handler project_name = data_handler.get_issue_tracker_name() issue_project_config = issue_tracker_config.get(project_name) if not issue_project_config: raise ValueError('Issue tracker for {} does not exist'.format(project_name)) constructor = _ISSUE_TRACKER_CONSTRUCTORS.get(issue_project_config['type']) if not constructor: raise ValueError('Invalid issue tracker type: ' + issue_project_config['type']) return constructor(project_name, issue_project_config)
def test_get_default(self): """Test getting default issue tracker.""" os.environ['ISSUE_TRACKER'] = 'test-issue-tracker' self.assertEqual('test-issue-tracker', data_handler.get_issue_tracker_name('job'))
def get_testcase_detail(testcase): """Get testcase detail for rendering the testcase detail page.""" config = db_config.get() crash_address = testcase.crash_address crash_state = testcase.crash_state crash_state_lines = crash_state.strip().splitlines() crash_type = data_handler.get_crash_type_string(testcase) reproduction_help_url = data_handler.get_reproduction_help_url( testcase, config) external_user = not access.has_access(job_type=testcase.job_type) issue_url = issue_tracker_utils.get_issue_url(testcase) metadata = testcase.get_metadata() original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys) minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys) has_issue_tracker = bool(data_handler.get_issue_tracker_name()) if not testcase.regression: regression = 'Pending' elif testcase.regression == 'NA': regression = 'NA' else: regression = _get_revision_range_html_from_string(testcase.job_type, testcase.regression) fixed_full = None if 'progression_pending' in metadata: fixed = 'Pending' elif not testcase.fixed: fixed = 'NO' elif testcase.fixed == 'NA': fixed = 'NA' elif testcase.fixed == 'Yes': fixed = 'YES' else: fixed = 'YES' fixed_full = _get_revision_range_html_from_string(testcase.job_type, testcase.fixed) last_tested = None last_tested_revision = ( metadata.get('last_tested_revision') or testcase.crash_revision) if last_tested_revision: last_tested = _get_revision_range_html(testcase.job_type, last_tested_revision) crash_revision = testcase.crash_revision crash_revisions_dict = revisions.get_component_revisions_dict( crash_revision, testcase.job_type) crash_stacktrace = data_handler.get_stacktrace(testcase) crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type, crash_revisions_dict) crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines, crash_type) crash_stacktrace_preview_lines = _preview_stacktrace(crash_stacktrace) second_crash_stacktrace_revision = metadata.get( 'second_crash_stacktrace_revision') second_crash_stacktrace_revisions_dict = ( revisions.get_component_revisions_dict(second_crash_stacktrace_revision, testcase.job_type)) second_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute='second_crash_stacktrace') second_crash_stacktrace = filter_stacktrace( second_crash_stacktrace, testcase.crash_type, second_crash_stacktrace_revisions_dict) second_crash_stacktrace = convert_to_lines(second_crash_stacktrace, crash_state_lines, crash_type) second_crash_stacktrace_preview_lines = _preview_stacktrace( second_crash_stacktrace) last_tested_crash_revision = metadata.get('last_tested_crash_revision') last_tested_crash_revisions_dict = revisions.get_component_revisions_dict( last_tested_crash_revision, testcase.job_type) last_tested_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute='last_tested_crash_stacktrace') last_tested_crash_stacktrace = filter_stacktrace( last_tested_crash_stacktrace, testcase.crash_type, last_tested_crash_revisions_dict) last_tested_crash_stacktrace = convert_to_lines(last_tested_crash_stacktrace, crash_state_lines, crash_type) last_tested_crash_stacktrace_preview_lines = _preview_stacktrace( last_tested_crash_stacktrace) privileged_user = access.has_access(need_privileged_access=True) # Fix build url link. |storage.cloud.google.com| takes care of using the # right set of authentication credentials needed to access the link. if 'build_url' in metadata: metadata['build_url'] = metadata['build_url'].replace( 'gs://', 'https://storage.cloud.google.com/') pending_blame_task = ( testcase.has_blame() and 'blame_pending' in metadata and metadata['blame_pending']) pending_impact_task = ( testcase.has_impacts() and not testcase.is_impact_set_flag) pending_minimize_task = not testcase.minimized_keys pending_progression_task = ('progression_pending' in metadata and metadata['progression_pending']) pending_regression_task = not testcase.regression pending_stack_task = testcase.last_tested_crash_stacktrace == 'Pending' needs_refresh = ( testcase.status == 'Pending' or ((testcase.status == 'Processed' or testcase.status == 'Duplicate') and (pending_blame_task or pending_impact_task or pending_minimize_task or pending_progression_task or pending_regression_task or pending_stack_task))) if data_types.SecuritySeverity.is_valid(testcase.security_severity): security_severity = severity_analyzer.severity_to_string( testcase.security_severity) else: security_severity = None auto_delete_timestamp = None auto_close_timestamp = None if testcase.one_time_crasher_flag: last_crash_time = ( crash_stats.get_last_crash_time(testcase) or testcase.timestamp) # Set auto-delete timestamp for unreproducible testcases with # no associated bug. if not testcase.bug_information: auto_delete_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE)) # Set auto-close timestamp for unreproducible testcases with # an associated bug. if testcase.open and testcase.bug_information: auto_close_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)) memory_tool_display_string = environment.get_memory_tool_display_string( testcase.job_type) memory_tool_display_label = memory_tool_display_string.split(':')[0] memory_tool_display_value = memory_tool_display_string.split(':')[1].strip() helpers.log('Testcase %s' % testcase.key.id(), helpers.VIEW_OPERATION) return { 'id': testcase.key.id(), 'crash_type': crash_type, 'crash_address': crash_address, 'crash_state': crash_state, # Used by reproduce tool. 'crash_state_lines': crash_state_lines, 'crash_revision': testcase.crash_revision, 'csrf_token': form.generate_csrf_token(), 'external_user': external_user, 'footer': testcase.comments, 'fixed': fixed, 'fixed_full': fixed_full, 'issue_url': issue_url, 'is_admin': auth.is_current_user_admin(), 'metadata': metadata, 'minimized_testcase_size': minimized_testcase_size, 'needs_refresh': needs_refresh, 'original_testcase_size': original_testcase_size, 'privileged_user': privileged_user, 'regression': regression, 'crash_stacktrace': { 'lines': crash_stacktrace, 'preview_lines': crash_stacktrace_preview_lines, 'revision': revisions.get_real_revision( crash_revision, testcase.job_type, display=True) }, 'second_crash_stacktrace': { 'lines': second_crash_stacktrace, 'preview_lines': second_crash_stacktrace_preview_lines, 'revision': revisions.get_real_revision( second_crash_stacktrace_revision, testcase.job_type, display=True) }, 'last_tested_crash_stacktrace': { 'lines': last_tested_crash_stacktrace, 'preview_lines': last_tested_crash_stacktrace_preview_lines, 'revision': revisions.get_real_revision( last_tested_crash_revision, testcase.job_type, display=True) }, 'security_severity': security_severity, 'security_severities': data_types.SecuritySeverity.list(), 'stats': { 'min_hour': crash_stats.get_min_hour(), 'max_hour': crash_stats.get_max_hour(), }, 'suspected_cls': _parse_suspected_cls(metadata.get('predator_result')), 'testcase': testcase, 'timestamp': utils.utc_datetime_to_timestamp(testcase.timestamp), 'show_blame': testcase.has_blame(), 'show_impact': testcase.has_impacts(), 'impacts_production': testcase.impacts_production(), 'find_similar_issues_options': FIND_SIMILAR_ISSUES_OPTIONS, 'auto_delete_timestamp': auto_delete_timestamp, 'auto_close_timestamp': auto_close_timestamp, 'memory_tool_display_label': memory_tool_display_label, 'memory_tool_display_value': memory_tool_display_value, 'last_tested': last_tested, 'is_admin_or_not_oss_fuzz': is_admin_or_not_oss_fuzz(), 'has_issue_tracker': has_issue_tracker, 'reproduction_help_url': reproduction_help_url, 'is_local_development': environment.is_running_on_app_engine_development(), }
def _get_issue_tracker_project_name(testcase=None): """Return issue tracker project name given a testcase or default.""" from datastore import data_handler job_type = testcase.job_type if testcase else None return data_handler.get_issue_tracker_name(job_type)
def get_testcase_detail(testcase): """Get testcase detail for rendering the testcase detail page.""" config = db_config.get() crash_address = testcase.crash_address crash_state = testcase.crash_state crash_state_lines = crash_state.strip().splitlines() crash_type = data_handler.get_crash_type_string(testcase) external_user = not access.has_access(job_type=testcase.job_type) issue_url = issue_tracker_utils.get_issue_url(testcase) metadata = testcase.get_metadata() original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys) minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys) has_issue_tracker = bool(data_handler.get_issue_tracker_name()) fuzzer_display = data_handler.get_fuzzer_display(testcase) formatted_reproduction_help = _format_reproduction_help( data_handler.get_formatted_reproduction_help(testcase)) # When we have a HELP_TEMPLATE, ignore any default values set for HELP_URL. if not formatted_reproduction_help: reproduction_help_url = data_handler.get_reproduction_help_url( testcase, config) else: reproduction_help_url = None if not testcase.regression: regression = "Pending" elif testcase.regression == "NA": regression = "NA" else: regression = _get_revision_range_html_from_string( testcase.job_type, testcase.regression) fixed_full = None if "progression_pending" in metadata: fixed = "Pending" elif not testcase.fixed: fixed = "NO" elif testcase.fixed == "NA": fixed = "NA" elif testcase.fixed == "Yes": fixed = "YES" else: fixed = "YES" fixed_full = _get_revision_range_html_from_string( testcase.job_type, testcase.fixed) last_tested = None last_tested_revision = (metadata.get("last_tested_revision") or testcase.crash_revision) if last_tested_revision: last_tested = _get_revision_range_html(testcase.job_type, last_tested_revision) crash_revision = testcase.crash_revision crash_revisions_dict = revisions.get_component_revisions_dict( crash_revision, testcase.job_type) crash_stacktrace = data_handler.get_stacktrace(testcase) crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type, crash_revisions_dict) crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines, crash_type) crash_stacktrace_preview_lines = _preview_stacktrace(crash_stacktrace) last_tested_crash_revision = metadata.get("last_tested_crash_revision") last_tested_crash_revisions_dict = revisions.get_component_revisions_dict( last_tested_crash_revision, testcase.job_type) last_tested_crash_stacktrace = data_handler.get_stacktrace( testcase, stack_attribute="last_tested_crash_stacktrace") last_tested_crash_stacktrace = filter_stacktrace( last_tested_crash_stacktrace, testcase.crash_type, last_tested_crash_revisions_dict, ) last_tested_crash_stacktrace = convert_to_lines( last_tested_crash_stacktrace, crash_state_lines, crash_type) last_tested_crash_stacktrace_preview_lines = _preview_stacktrace( last_tested_crash_stacktrace) privileged_user = access.has_access(need_privileged_access=True) # Fix build url link. |storage.cloud.google.com| takes care of using the # right set of authentication credentials needed to access the link. if "build_url" in metadata: metadata["build_url"] = metadata["build_url"].replace( "gs://", "https://storage.cloud.google.com/") pending_blame_task = (testcase.has_blame() and "blame_pending" in metadata and metadata["blame_pending"]) pending_impact_task = testcase.has_impacts( ) and not testcase.is_impact_set_flag pending_minimize_task = not testcase.minimized_keys pending_progression_task = ("progression_pending" in metadata and metadata["progression_pending"]) pending_regression_task = not testcase.regression pending_stack_task = testcase.last_tested_crash_stacktrace == "Pending" needs_refresh = testcase.status == "Pending" or ( (testcase.status == "Processed" or testcase.status == "Duplicate") and (pending_blame_task or pending_impact_task or pending_minimize_task or pending_progression_task or pending_regression_task or pending_stack_task)) if data_types.SecuritySeverity.is_valid(testcase.security_severity): security_severity = severity_analyzer.severity_to_string( testcase.security_severity) else: security_severity = None auto_delete_timestamp = None auto_close_timestamp = None if testcase.one_time_crasher_flag: last_crash_time = (crash_stats.get_last_crash_time(testcase) or testcase.timestamp) # Set auto-delete timestamp for unreproducible testcases with # no associated bug. if not testcase.bug_information: auto_delete_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE)) # Set auto-close timestamp for unreproducible testcases with # an associated bug. if testcase.open and testcase.bug_information: auto_close_timestamp = utils.utc_datetime_to_timestamp( last_crash_time + datetime.timedelta( days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE)) memory_tool_display_string = environment.get_memory_tool_display_string( testcase.job_type) memory_tool_display_label = memory_tool_display_string.split(":")[0] memory_tool_display_value = memory_tool_display_string.split( ":")[1].strip() helpers.log("Testcase %s" % testcase.key.id(), helpers.VIEW_OPERATION) return { "id": testcase.key.id(), "crash_type": crash_type, "crash_address": crash_address, "crash_state": crash_state, # Used by reproduce tool. "crash_state_lines": crash_state_lines, "crash_revision": testcase.crash_revision, "csrf_token": form.generate_csrf_token(), "external_user": external_user, "footer": testcase.comments, "formatted_reproduction_help": formatted_reproduction_help, "fixed": fixed, "fixed_full": fixed_full, "issue_url": issue_url, "is_admin": auth.is_current_user_admin(), "metadata": metadata, "minimized_testcase_size": minimized_testcase_size, "needs_refresh": needs_refresh, "original_testcase_size": original_testcase_size, "privileged_user": privileged_user, "regression": regression, "crash_stacktrace": { "lines": crash_stacktrace, "preview_lines": crash_stacktrace_preview_lines, "revision": revisions.get_real_revision(crash_revision, testcase.job_type, display=True), }, "last_tested_crash_stacktrace": { "lines": last_tested_crash_stacktrace, "preview_lines": last_tested_crash_stacktrace_preview_lines, "revision": revisions.get_real_revision(last_tested_crash_revision, testcase.job_type, display=True), }, "security_severity": security_severity, "security_severities": data_types.SecuritySeverity.list(), "stats": { "min_hour": crash_stats.get_min_hour(), "max_hour": crash_stats.get_max_hour(), }, "suspected_cls": _parse_suspected_cls(metadata.get("predator_result")), "testcase": testcase, "timestamp": utils.utc_datetime_to_timestamp(testcase.timestamp), "show_blame": testcase.has_blame(), "show_impact": testcase.has_impacts(), "impacts_production": testcase.impacts_production(), "find_similar_issues_options": FIND_SIMILAR_ISSUES_OPTIONS, "auto_delete_timestamp": auto_delete_timestamp, "auto_close_timestamp": auto_close_timestamp, "memory_tool_display_label": memory_tool_display_label, "memory_tool_display_value": memory_tool_display_value, "last_tested": last_tested, "is_admin_or_not_oss_fuzz": is_admin_or_not_oss_fuzz(), "has_issue_tracker": has_issue_tracker, "reproduction_help_url": reproduction_help_url, "is_local_development": environment.is_running_on_app_engine_development(), "fuzzer_display": vars(fuzzer_display), }