def test_get_from_job(self):
"""Test getting from job."""
data_types.Job(name='job',
environment_string=('ISSUE_TRACKER = from_job\n'
'HELP_URL = help_url\n')).put()
self.assertEqual('from_job',
data_handler.get_issue_tracker_name('job'))
def get(self):
"""Handles get request."""
email = helpers.get_user_email()
if not email:
raise helpers.AccessDeniedException()
is_privileged_or_domain_user = access.has_access(
need_privileged_access=False)
if is_privileged_or_domain_user or _is_uploader_allowed(email):
# Privileged, domain and upload users can see all job and fuzzer names.
allowed_jobs = data_handler.get_all_job_type_names()
allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children(
include_parents=True)
else:
# Check if this is an external user with access to certain fuzzers/jobs.
allowed_jobs = external_users.allowed_jobs_for_user(email)
allowed_fuzzers = external_users.allowed_fuzzers_for_user(
email, include_from_jobs=True)
if not allowed_fuzzers and not allowed_jobs:
raise helpers.AccessDeniedException()
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
result, params = get_result()
return self.render(
'upload.html', {
'fieldValues': {
'blackboxFuzzers':
filter_blackbox_fuzzers(allowed_fuzzers),
'jobs':
allowed_jobs,
'libfuzzerTargets':
filter_target_names(allowed_fuzzers, 'libFuzzer'),
'aflTargets':
filter_target_names(allowed_fuzzers, 'afl'),
'honggfuzzTargets':
filter_target_names(allowed_fuzzers, 'honggfuzz'),
'isChromium':
utils.is_chromium(),
'sandboxedJobs':
data_types.INTERNAL_SANDBOXED_JOB_TYPES,
'csrfToken':
form.generate_csrf_token(),
'isExternalUser':
not is_privileged_or_domain_user,
'uploadInfo':
gcs.prepare_blob_upload()._asdict(),
'hasIssueTracker':
has_issue_tracker,
},
'params': params,
'result': result
})
def get(self):
"""Handles get request."""
email = helpers.get_user_email()
if not email:
raise helpers.AccessDeniedException()
is_privileged_or_domain_user = access.has_access(
need_privileged_access=False)
if is_privileged_or_domain_user or _is_uploader_allowed(email):
# Privileged, domain and upload users can see all job and fuzzer names.
allowed_jobs = data_handler.get_all_job_type_names()
allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children(
include_parents=True)
else:
# Check if this is an external user with access to certain fuzzers/jobs.
allowed_jobs = external_users.allowed_jobs_for_user(email)
allowed_fuzzers = external_users.allowed_fuzzers_for_user(
email, include_from_jobs=True)
if not allowed_fuzzers and not allowed_jobs:
raise helpers.AccessDeniedException()
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
result, params = get_result(self)
self.render(
"upload.html",
{
"fieldValues": {
"jobs":
allowed_jobs,
"libfuzzerTargets":
filter_target_names(allowed_fuzzers, "libFuzzer"),
"aflTargets":
filter_target_names(allowed_fuzzers, "afl"),
"isChromium":
utils.is_chromium(),
"sandboxedJobs":
data_types.INTERNAL_SANDBOXED_JOB_TYPES,
"csrfToken":
form.generate_csrf_token(),
"isExternalUser":
not is_privileged_or_domain_user,
"uploadInfo":
gcs.prepare_blob_upload()._asdict(),
"hasIssueTracker":
has_issue_tracker,
},
"params": params,
"result": result,
},
)
def get_issue_tracker(tracker_type, project_name=None, use_cache=False):
"""Get the issue tracker with the given type and name."""
# TODO(ochang): Actually use `tracker_type`.
assert tracker_type == 'monorail'
if not project_name:
from datastore import data_handler
project_name = data_handler.get_issue_tracker_name()
itm = _get_issue_tracker_manager_for_project(project_name,
use_cache=use_cache)
if itm is None:
return None
return monorail.IssueTracker(itm)
def get_issue_tracker(project_name=None):
"""Get the issue tracker with the given type and name."""
issue_tracker_config = local_config.IssueTrackerConfig()
if not project_name:
from datastore import data_handler
project_name = data_handler.get_issue_tracker_name()
issue_project_config = issue_tracker_config.get(project_name)
if not issue_project_config:
raise ValueError('Issue tracker for {} does not exist'.format(project_name))
constructor = _ISSUE_TRACKER_CONSTRUCTORS.get(issue_project_config['type'])
if not constructor:
raise ValueError('Invalid issue tracker type: ' +
issue_project_config['type'])
return constructor(project_name, issue_project_config)
def test_get_default(self):
"""Test getting default issue tracker."""
os.environ['ISSUE_TRACKER'] = 'test-issue-tracker'
self.assertEqual('test-issue-tracker',
data_handler.get_issue_tracker_name('job'))
def get_testcase_detail(testcase):
"""Get testcase detail for rendering the testcase detail page."""
config = db_config.get()
crash_address = testcase.crash_address
crash_state = testcase.crash_state
crash_state_lines = crash_state.strip().splitlines()
crash_type = data_handler.get_crash_type_string(testcase)
reproduction_help_url = data_handler.get_reproduction_help_url(
testcase, config)
external_user = not access.has_access(job_type=testcase.job_type)
issue_url = issue_tracker_utils.get_issue_url(testcase)
metadata = testcase.get_metadata()
original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys)
minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys)
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
if not testcase.regression:
regression = 'Pending'
elif testcase.regression == 'NA':
regression = 'NA'
else:
regression = _get_revision_range_html_from_string(testcase.job_type,
testcase.regression)
fixed_full = None
if 'progression_pending' in metadata:
fixed = 'Pending'
elif not testcase.fixed:
fixed = 'NO'
elif testcase.fixed == 'NA':
fixed = 'NA'
elif testcase.fixed == 'Yes':
fixed = 'YES'
else:
fixed = 'YES'
fixed_full = _get_revision_range_html_from_string(testcase.job_type,
testcase.fixed)
last_tested = None
last_tested_revision = (
metadata.get('last_tested_revision') or testcase.crash_revision)
if last_tested_revision:
last_tested = _get_revision_range_html(testcase.job_type,
last_tested_revision)
crash_revision = testcase.crash_revision
crash_revisions_dict = revisions.get_component_revisions_dict(
crash_revision, testcase.job_type)
crash_stacktrace = data_handler.get_stacktrace(testcase)
crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type,
crash_revisions_dict)
crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines,
crash_type)
crash_stacktrace_preview_lines = _preview_stacktrace(crash_stacktrace)
second_crash_stacktrace_revision = metadata.get(
'second_crash_stacktrace_revision')
second_crash_stacktrace_revisions_dict = (
revisions.get_component_revisions_dict(second_crash_stacktrace_revision,
testcase.job_type))
second_crash_stacktrace = data_handler.get_stacktrace(
testcase, stack_attribute='second_crash_stacktrace')
second_crash_stacktrace = filter_stacktrace(
second_crash_stacktrace, testcase.crash_type,
second_crash_stacktrace_revisions_dict)
second_crash_stacktrace = convert_to_lines(second_crash_stacktrace,
crash_state_lines, crash_type)
second_crash_stacktrace_preview_lines = _preview_stacktrace(
second_crash_stacktrace)
last_tested_crash_revision = metadata.get('last_tested_crash_revision')
last_tested_crash_revisions_dict = revisions.get_component_revisions_dict(
last_tested_crash_revision, testcase.job_type)
last_tested_crash_stacktrace = data_handler.get_stacktrace(
testcase, stack_attribute='last_tested_crash_stacktrace')
last_tested_crash_stacktrace = filter_stacktrace(
last_tested_crash_stacktrace, testcase.crash_type,
last_tested_crash_revisions_dict)
last_tested_crash_stacktrace = convert_to_lines(last_tested_crash_stacktrace,
crash_state_lines, crash_type)
last_tested_crash_stacktrace_preview_lines = _preview_stacktrace(
last_tested_crash_stacktrace)
privileged_user = access.has_access(need_privileged_access=True)
# Fix build url link. |storage.cloud.google.com| takes care of using the
# right set of authentication credentials needed to access the link.
if 'build_url' in metadata:
metadata['build_url'] = metadata['build_url'].replace(
'gs://', 'https://storage.cloud.google.com/')
pending_blame_task = (
testcase.has_blame() and 'blame_pending' in metadata and
metadata['blame_pending'])
pending_impact_task = (
testcase.has_impacts() and not testcase.is_impact_set_flag)
pending_minimize_task = not testcase.minimized_keys
pending_progression_task = ('progression_pending' in metadata and
metadata['progression_pending'])
pending_regression_task = not testcase.regression
pending_stack_task = testcase.last_tested_crash_stacktrace == 'Pending'
needs_refresh = (
testcase.status == 'Pending' or
((testcase.status == 'Processed' or testcase.status == 'Duplicate') and
(pending_blame_task or pending_impact_task or pending_minimize_task or
pending_progression_task or pending_regression_task or
pending_stack_task)))
if data_types.SecuritySeverity.is_valid(testcase.security_severity):
security_severity = severity_analyzer.severity_to_string(
testcase.security_severity)
else:
security_severity = None
auto_delete_timestamp = None
auto_close_timestamp = None
if testcase.one_time_crasher_flag:
last_crash_time = (
crash_stats.get_last_crash_time(testcase) or testcase.timestamp)
# Set auto-delete timestamp for unreproducible testcases with
# no associated bug.
if not testcase.bug_information:
auto_delete_timestamp = utils.utc_datetime_to_timestamp(
last_crash_time + datetime.timedelta(
days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE))
# Set auto-close timestamp for unreproducible testcases with
# an associated bug.
if testcase.open and testcase.bug_information:
auto_close_timestamp = utils.utc_datetime_to_timestamp(
last_crash_time + datetime.timedelta(
days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE))
memory_tool_display_string = environment.get_memory_tool_display_string(
testcase.job_type)
memory_tool_display_label = memory_tool_display_string.split(':')[0]
memory_tool_display_value = memory_tool_display_string.split(':')[1].strip()
helpers.log('Testcase %s' % testcase.key.id(), helpers.VIEW_OPERATION)
return {
'id':
testcase.key.id(),
'crash_type':
crash_type,
'crash_address':
crash_address,
'crash_state':
crash_state, # Used by reproduce tool.
'crash_state_lines':
crash_state_lines,
'crash_revision':
testcase.crash_revision,
'csrf_token':
form.generate_csrf_token(),
'external_user':
external_user,
'footer':
testcase.comments,
'fixed':
fixed,
'fixed_full':
fixed_full,
'issue_url':
issue_url,
'is_admin':
auth.is_current_user_admin(),
'metadata':
metadata,
'minimized_testcase_size':
minimized_testcase_size,
'needs_refresh':
needs_refresh,
'original_testcase_size':
original_testcase_size,
'privileged_user':
privileged_user,
'regression':
regression,
'crash_stacktrace': {
'lines':
crash_stacktrace,
'preview_lines':
crash_stacktrace_preview_lines,
'revision':
revisions.get_real_revision(
crash_revision, testcase.job_type, display=True)
},
'second_crash_stacktrace': {
'lines':
second_crash_stacktrace,
'preview_lines':
second_crash_stacktrace_preview_lines,
'revision':
revisions.get_real_revision(
second_crash_stacktrace_revision,
testcase.job_type,
display=True)
},
'last_tested_crash_stacktrace': {
'lines':
last_tested_crash_stacktrace,
'preview_lines':
last_tested_crash_stacktrace_preview_lines,
'revision':
revisions.get_real_revision(
last_tested_crash_revision, testcase.job_type, display=True)
},
'security_severity':
security_severity,
'security_severities':
data_types.SecuritySeverity.list(),
'stats': {
'min_hour': crash_stats.get_min_hour(),
'max_hour': crash_stats.get_max_hour(),
},
'suspected_cls':
_parse_suspected_cls(metadata.get('predator_result')),
'testcase':
testcase,
'timestamp':
utils.utc_datetime_to_timestamp(testcase.timestamp),
'show_blame':
testcase.has_blame(),
'show_impact':
testcase.has_impacts(),
'impacts_production':
testcase.impacts_production(),
'find_similar_issues_options':
FIND_SIMILAR_ISSUES_OPTIONS,
'auto_delete_timestamp':
auto_delete_timestamp,
'auto_close_timestamp':
auto_close_timestamp,
'memory_tool_display_label':
memory_tool_display_label,
'memory_tool_display_value':
memory_tool_display_value,
'last_tested':
last_tested,
'is_admin_or_not_oss_fuzz':
is_admin_or_not_oss_fuzz(),
'has_issue_tracker':
has_issue_tracker,
'reproduction_help_url':
reproduction_help_url,
'is_local_development':
environment.is_running_on_app_engine_development(),
}
def _get_issue_tracker_project_name(testcase=None):
"""Return issue tracker project name given a testcase or default."""
from datastore import data_handler
job_type = testcase.job_type if testcase else None
return data_handler.get_issue_tracker_name(job_type)
def get_testcase_detail(testcase):
"""Get testcase detail for rendering the testcase detail page."""
config = db_config.get()
crash_address = testcase.crash_address
crash_state = testcase.crash_state
crash_state_lines = crash_state.strip().splitlines()
crash_type = data_handler.get_crash_type_string(testcase)
external_user = not access.has_access(job_type=testcase.job_type)
issue_url = issue_tracker_utils.get_issue_url(testcase)
metadata = testcase.get_metadata()
original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys)
minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys)
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
fuzzer_display = data_handler.get_fuzzer_display(testcase)
formatted_reproduction_help = _format_reproduction_help(
data_handler.get_formatted_reproduction_help(testcase))
# When we have a HELP_TEMPLATE, ignore any default values set for HELP_URL.
if not formatted_reproduction_help:
reproduction_help_url = data_handler.get_reproduction_help_url(
testcase, config)
else:
reproduction_help_url = None
if not testcase.regression:
regression = "Pending"
elif testcase.regression == "NA":
regression = "NA"
else:
regression = _get_revision_range_html_from_string(
testcase.job_type, testcase.regression)
fixed_full = None
if "progression_pending" in metadata:
fixed = "Pending"
elif not testcase.fixed:
fixed = "NO"
elif testcase.fixed == "NA":
fixed = "NA"
elif testcase.fixed == "Yes":
fixed = "YES"
else:
fixed = "YES"
fixed_full = _get_revision_range_html_from_string(
testcase.job_type, testcase.fixed)
last_tested = None
last_tested_revision = (metadata.get("last_tested_revision")
or testcase.crash_revision)
if last_tested_revision:
last_tested = _get_revision_range_html(testcase.job_type,
last_tested_revision)
crash_revision = testcase.crash_revision
crash_revisions_dict = revisions.get_component_revisions_dict(
crash_revision, testcase.job_type)
crash_stacktrace = data_handler.get_stacktrace(testcase)
crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type,
crash_revisions_dict)
crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines,
crash_type)
crash_stacktrace_preview_lines = _preview_stacktrace(crash_stacktrace)
last_tested_crash_revision = metadata.get("last_tested_crash_revision")
last_tested_crash_revisions_dict = revisions.get_component_revisions_dict(
last_tested_crash_revision, testcase.job_type)
last_tested_crash_stacktrace = data_handler.get_stacktrace(
testcase, stack_attribute="last_tested_crash_stacktrace")
last_tested_crash_stacktrace = filter_stacktrace(
last_tested_crash_stacktrace,
testcase.crash_type,
last_tested_crash_revisions_dict,
)
last_tested_crash_stacktrace = convert_to_lines(
last_tested_crash_stacktrace, crash_state_lines, crash_type)
last_tested_crash_stacktrace_preview_lines = _preview_stacktrace(
last_tested_crash_stacktrace)
privileged_user = access.has_access(need_privileged_access=True)
# Fix build url link. |storage.cloud.google.com| takes care of using the
# right set of authentication credentials needed to access the link.
if "build_url" in metadata:
metadata["build_url"] = metadata["build_url"].replace(
"gs://", "https://storage.cloud.google.com/")
pending_blame_task = (testcase.has_blame() and "blame_pending" in metadata
and metadata["blame_pending"])
pending_impact_task = testcase.has_impacts(
) and not testcase.is_impact_set_flag
pending_minimize_task = not testcase.minimized_keys
pending_progression_task = ("progression_pending" in metadata
and metadata["progression_pending"])
pending_regression_task = not testcase.regression
pending_stack_task = testcase.last_tested_crash_stacktrace == "Pending"
needs_refresh = testcase.status == "Pending" or (
(testcase.status == "Processed" or testcase.status == "Duplicate") and
(pending_blame_task or pending_impact_task or pending_minimize_task
or pending_progression_task or pending_regression_task
or pending_stack_task))
if data_types.SecuritySeverity.is_valid(testcase.security_severity):
security_severity = severity_analyzer.severity_to_string(
testcase.security_severity)
else:
security_severity = None
auto_delete_timestamp = None
auto_close_timestamp = None
if testcase.one_time_crasher_flag:
last_crash_time = (crash_stats.get_last_crash_time(testcase)
or testcase.timestamp)
# Set auto-delete timestamp for unreproducible testcases with
# no associated bug.
if not testcase.bug_information:
auto_delete_timestamp = utils.utc_datetime_to_timestamp(
last_crash_time + datetime.timedelta(
days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE))
# Set auto-close timestamp for unreproducible testcases with
# an associated bug.
if testcase.open and testcase.bug_information:
auto_close_timestamp = utils.utc_datetime_to_timestamp(
last_crash_time + datetime.timedelta(
days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE))
memory_tool_display_string = environment.get_memory_tool_display_string(
testcase.job_type)
memory_tool_display_label = memory_tool_display_string.split(":")[0]
memory_tool_display_value = memory_tool_display_string.split(
":")[1].strip()
helpers.log("Testcase %s" % testcase.key.id(), helpers.VIEW_OPERATION)
return {
"id": testcase.key.id(),
"crash_type": crash_type,
"crash_address": crash_address,
"crash_state": crash_state, # Used by reproduce tool.
"crash_state_lines": crash_state_lines,
"crash_revision": testcase.crash_revision,
"csrf_token": form.generate_csrf_token(),
"external_user": external_user,
"footer": testcase.comments,
"formatted_reproduction_help": formatted_reproduction_help,
"fixed": fixed,
"fixed_full": fixed_full,
"issue_url": issue_url,
"is_admin": auth.is_current_user_admin(),
"metadata": metadata,
"minimized_testcase_size": minimized_testcase_size,
"needs_refresh": needs_refresh,
"original_testcase_size": original_testcase_size,
"privileged_user": privileged_user,
"regression": regression,
"crash_stacktrace": {
"lines":
crash_stacktrace,
"preview_lines":
crash_stacktrace_preview_lines,
"revision":
revisions.get_real_revision(crash_revision,
testcase.job_type,
display=True),
},
"last_tested_crash_stacktrace": {
"lines":
last_tested_crash_stacktrace,
"preview_lines":
last_tested_crash_stacktrace_preview_lines,
"revision":
revisions.get_real_revision(last_tested_crash_revision,
testcase.job_type,
display=True),
},
"security_severity": security_severity,
"security_severities": data_types.SecuritySeverity.list(),
"stats": {
"min_hour": crash_stats.get_min_hour(),
"max_hour": crash_stats.get_max_hour(),
},
"suspected_cls": _parse_suspected_cls(metadata.get("predator_result")),
"testcase": testcase,
"timestamp": utils.utc_datetime_to_timestamp(testcase.timestamp),
"show_blame": testcase.has_blame(),
"show_impact": testcase.has_impacts(),
"impacts_production": testcase.impacts_production(),
"find_similar_issues_options": FIND_SIMILAR_ISSUES_OPTIONS,
"auto_delete_timestamp": auto_delete_timestamp,
"auto_close_timestamp": auto_close_timestamp,
"memory_tool_display_label": memory_tool_display_label,
"memory_tool_display_value": memory_tool_display_value,
"last_tested": last_tested,
"is_admin_or_not_oss_fuzz": is_admin_or_not_oss_fuzz(),
"has_issue_tracker": has_issue_tracker,
"reproduction_help_url": reproduction_help_url,
"is_local_development":
environment.is_running_on_app_engine_development(),
"fuzzer_display": vars(fuzzer_display),
}