def to_dict(entity):
"""Convert a db.Model instance to a dict."""
entity_dict = entity.to_dict()
entity_dict['id'] = entity.key.id()
for k, v in six.iteritems(entity_dict):
if isinstance(v, datetime.datetime):
entity_dict[k] = utils.utc_datetime_to_timestamp(v)
return entity_dict
def to_pubsub_message(self):
"""Convert the task to a pubsub message."""
attributes = {
'command': self.command,
'argument': str(self.argument),
'job': self.job,
}
if self.eta:
attributes['eta'] = str(utils.utc_datetime_to_timestamp(self.eta))
return pubsub.Message(attributes=attributes)
def get_result():
"""Get the result."""
params = dict(request.iterparams())
page = helpers.cast(request.get('page') or 1, int, "'page' is not an int.")
query = datastore_query.Query(data_types.TestcaseUploadMetadata)
query.order('timestamp', is_desc=True)
if not access.has_access(need_privileged_access=True):
query.filter('uploader_email', helpers.get_user_email())
params['permission'] = {'uploaderEmail': helpers.get_user_email()}
entities, total_pages, total_items, has_more = query.fetch_page(
page=page, page_size=PAGE_SIZE, projection=None, more_limit=MORE_LIMIT)
items = []
for entity in entities:
items.append({
'timestamp':
utils.utc_datetime_to_timestamp(entity.timestamp),
'testcaseId':
entity.testcase_id,
'uploaderEmail':
entity.uploader_email,
'filename':
entity.filename,
'bundled':
entity.bundled,
'pathInArchive':
entity.path_in_archive,
'status':
entity.status
})
attach_testcases(items)
result = {
'hasMore': has_more,
'items': items,
'page': page,
'pageSize': PAGE_SIZE,
'totalItems': total_items,
'totalPages': total_pages,
}
return result, params
def test_get(self):
"""Test get."""
mtime = datetime.datetime(2019, 1, 1)
mtime_seconds = utils.utc_datetime_to_timestamp(mtime)
self.fs.create_file('/local/test-bucket/objects/a',
contents='a').st_mtime = mtime_seconds
self.fs.create_file('/local/test-bucket/metadata/a',
contents='{"key": "value"}')
self.assertDictEqual(
{
'bucket': 'test-bucket',
'name': 'a',
'size': 1,
'updated': mtime,
'metadata': {
'key': 'value'
},
}, self.provider.get('gs://test-bucket/a'))
def get_result():
"""Get the result for the testcase list page."""
params = dict(request.iterparams())
page = helpers.cast(request.get('page') or 1, int, "'page' is not an int.")
query = datastore_query.Query(data_types.Testcase)
crash_access.add_scope(query, params, 'security_flag', 'job_type',
'fuzzer_name_indices')
add_filters(query, params)
testcases, total_pages, total_items, has_more = query.fetch_page(
page=page,
page_size=PAGE_SIZE,
projection=FIELDS,
more_limit=MORE_LIMIT)
items = []
for testcase in testcases:
regression_range = ''
fixed_range = ''
if testcase.regression and testcase.regression != 'NA':
regression_range = testcase.regression
if testcase.fixed and testcase.fixed != 'NA':
fixed_range = testcase.fixed
item = {
'id': testcase.key.id(),
'crashType': ' '.join(testcase.crash_type.splitlines()),
'crashStateLines': testcase.crash_state.strip().splitlines(),
'jobType': testcase.job_type,
'isClosed': not testcase.open,
'isFixed': testcase.fixed and testcase.fixed != 'NA',
'isReproducible': not testcase.one_time_crasher_flag,
'isSecurity': testcase.security_flag,
'isImpactSet': testcase.is_impact_set_flag,
'impacts': {
'extendedStable': testcase.impact_extended_stable_version,
'stable': testcase.impact_stable_version,
'beta': testcase.impact_beta_version,
'head': testcase.impact_head_version,
},
'regressionRange': regression_range,
'fixedRange': fixed_range,
'groupId': testcase.group_id,
'projectName': testcase.project_name,
'platform': testcase.platform,
'issueId': testcase.bug_information
or testcase.group_bug_information,
'showImpacts': testcase.has_impacts(),
'impactsProduction': testcase.impacts_production()
}
if testcase.timestamp:
item['timestamp'] = utils.utc_datetime_to_timestamp(
testcase.timestamp)
items.append(item)
helpers.log('Testcases', helpers.VIEW_OPERATION)
result = {
'hasMore': has_more,
'items': items,
'page': page,
'pageSize': PAGE_SIZE,
'totalItems': total_items,
'totalPages': total_pages,
}
return result, params
def test_list_blobs(self):
"""Test list_blobs."""
mtime = datetime.datetime(2019, 1, 1)
mtime_seconds = utils.utc_datetime_to_timestamp(mtime)
self.fs.create_file('/local/test-bucket/objects/a',
st_size=11).st_mtime = mtime_seconds
self.fs.create_file('/local/test-bucket/objects/b/c',
st_size=22).st_mtime = mtime_seconds
self.fs.create_file('/local/test-bucket/objects/b/d/e',
st_size=33).st_mtime = mtime_seconds
self.fs.create_file('/local/test-bucket/objects/f',
st_size=44).st_mtime = mtime_seconds
self.fs.create_file(
'/local/test-bucket/metadata/b/c',
contents='{"key":"value"}').st_mtime = mtime_seconds
result = list(self.provider.list_blobs('gs://test-bucket'))
six.assertCountEqual(self, [{
'bucket': 'test-bucket',
'name': 'a',
'updated': mtime,
'size': 11,
'metadata': {}
}, {
'bucket': 'test-bucket',
'name': 'f',
'updated': mtime,
'size': 44,
'metadata': {}
}, {
'bucket': 'test-bucket',
'name': 'b/c',
'updated': mtime,
'size': 22,
'metadata': {
'key': 'value'
}
}, {
'bucket': 'test-bucket',
'name': 'b/d/e',
'updated': mtime,
'size': 33,
'metadata': {}
}], result)
result = list(self.provider.list_blobs('gs://test-bucket/b'))
six.assertCountEqual(self, [{
'bucket': 'test-bucket',
'name': 'b/c',
'updated': mtime,
'size': 22,
'metadata': {
'key': 'value'
}
}, {
'bucket': 'test-bucket',
'name': 'b/d/e',
'updated': mtime,
'size': 33,
'metadata': {}
}], result)
result = list(self.provider.list_blobs('gs://test-bucket/b/d'))
six.assertCountEqual(self, [{
'bucket': 'test-bucket',
'name': 'b/d/e',
'updated': mtime,
'size': 33,
'metadata': {}
}], result)
result = list(
self.provider.list_blobs('gs://test-bucket/', recursive=False))
six.assertCountEqual(self, [{
'bucket': 'test-bucket',
'name': 'a',
'updated': mtime,
'size': 11,
'metadata': {}
}, {
'bucket': 'test-bucket',
'name': 'f',
'updated': mtime,
'size': 44,
'metadata': {}
}, {
'bucket': 'test-bucket',
'name': 'b',
}], result)
def get_testcase_detail(testcase):
"""Get testcase detail for rendering the testcase detail page."""
config = db_config.get()
crash_address = testcase.crash_address
crash_state = testcase.crash_state
crash_state_lines = crash_state.strip().splitlines()
crash_type = data_handler.get_crash_type_string(testcase)
external_user = not access.has_access(job_type=testcase.job_type)
issue_url = issue_tracker_utils.get_issue_url(testcase)
metadata = testcase.get_metadata()
original_testcase_size = _get_blob_size_string(testcase.fuzzed_keys)
minimized_testcase_size = _get_blob_size_string(testcase.minimized_keys)
has_issue_tracker = bool(data_handler.get_issue_tracker_name())
fuzzer_display = data_handler.get_fuzzer_display(testcase)
formatted_reproduction_help = _format_reproduction_help(
data_handler.get_formatted_reproduction_help(testcase))
# When we have a HELP_TEMPLATE, ignore any default values set for HELP_URL.
if not formatted_reproduction_help:
reproduction_help_url = data_handler.get_reproduction_help_url(
testcase, config)
else:
reproduction_help_url = None
if not testcase.regression:
regression = 'Pending'
elif testcase.regression == 'NA':
regression = 'NA'
else:
regression = _get_revision_range_html_from_string(
testcase.job_type, testcase.platform_id, testcase.regression)
fixed_full = None
if 'progression_pending' in metadata:
fixed = 'Pending'
elif not testcase.fixed:
fixed = 'NO'
elif testcase.fixed == 'NA':
fixed = 'NA'
elif testcase.fixed == 'Yes':
fixed = 'YES'
else:
fixed = 'YES'
fixed_full = _get_revision_range_html_from_string(
testcase.job_type, testcase.platform_id, testcase.fixed)
last_tested = None
last_tested_revision = (metadata.get('last_tested_revision')
or testcase.crash_revision)
if last_tested_revision:
last_tested = _get_revision_range_html(testcase.job_type,
testcase.platform_id,
last_tested_revision)
crash_revision = testcase.crash_revision
crash_revisions_dict = revisions.get_component_revisions_dict(
crash_revision, testcase.job_type, platform_id=testcase.platform_id)
crash_stacktrace = data_handler.get_stacktrace(testcase)
crash_stacktrace = filter_stacktrace(crash_stacktrace, testcase.crash_type,
crash_revisions_dict,
testcase.platform, testcase.job_type)
crash_stacktrace = convert_to_lines(crash_stacktrace, crash_state_lines,
crash_type)
last_tested_crash_revision = metadata.get('last_tested_crash_revision')
last_tested_crash_revisions_dict = revisions.get_component_revisions_dict(
last_tested_crash_revision,
testcase.job_type,
platform_id=testcase.platform_id)
last_tested_crash_stacktrace = data_handler.get_stacktrace(
testcase, stack_attribute='last_tested_crash_stacktrace')
last_tested_crash_stacktrace = filter_stacktrace(
last_tested_crash_stacktrace, testcase.crash_type,
last_tested_crash_revisions_dict, testcase.platform, testcase.job_type)
last_tested_crash_stacktrace = convert_to_lines(
last_tested_crash_stacktrace, crash_state_lines, crash_type)
privileged_user = access.has_access(need_privileged_access=True)
# Fix build url link. |storage.cloud.google.com| takes care of using the
# right set of authentication credentials needed to access the link.
if 'build_url' in metadata:
metadata['build_url'] = metadata['build_url'].replace(
'gs://', 'https://storage.cloud.google.com/')
pending_blame_task = (testcase.has_blame() and 'blame_pending' in metadata
and metadata['blame_pending'])
pending_impact_task = (testcase.has_impacts()
and not testcase.is_impact_set_flag)
pending_minimize_task = not testcase.minimized_keys
pending_progression_task = ('progression_pending' in metadata
and metadata['progression_pending'])
pending_regression_task = not testcase.regression
pending_stack_task = testcase.last_tested_crash_stacktrace == 'Pending'
needs_refresh = (testcase.status == 'Pending' or (
(testcase.status == 'Processed' or testcase.status == 'Duplicate') and
(pending_blame_task or pending_impact_task or pending_minimize_task
or pending_progression_task or pending_regression_task
or pending_stack_task)))
if data_types.SecuritySeverity.is_valid(testcase.security_severity):
security_severity = severity_analyzer.severity_to_string(
testcase.security_severity)
else:
security_severity = None
auto_delete_timestamp = None
auto_close_timestamp = None
if testcase.one_time_crasher_flag:
last_crash_time = (crash_stats.get_last_crash_time(testcase)
or testcase.timestamp)
# Set auto-delete timestamp for unreproducible testcases with
# no associated bug.
if not testcase.bug_information:
auto_delete_timestamp = utils.utc_datetime_to_timestamp(
last_crash_time + datetime.timedelta(
days=data_types.UNREPRODUCIBLE_TESTCASE_NO_BUG_DEADLINE))
# Set auto-close timestamp for unreproducible testcases with
# an associated bug.
if testcase.open and testcase.bug_information:
auto_close_timestamp = utils.utc_datetime_to_timestamp(
last_crash_time + datetime.timedelta(
days=data_types.UNREPRODUCIBLE_TESTCASE_WITH_BUG_DEADLINE))
memory_tool_display_string = environment.get_memory_tool_display_string(
testcase.job_type)
memory_tool_display_label = memory_tool_display_string.split(':')[0]
memory_tool_display_value = memory_tool_display_string.split(
':')[1].strip()
helpers.log('Testcase %s' % testcase.key.id(), helpers.VIEW_OPERATION)
return {
'id': testcase.key.id(),
'crash_type': crash_type,
'crash_address': crash_address,
'crash_state': crash_state, # Used by reproduce tool.
'crash_state_lines': crash_state_lines,
'crash_revision': testcase.crash_revision,
'csrf_token': form.generate_csrf_token(),
'external_user': external_user,
'footer': testcase.comments,
'formatted_reproduction_help': formatted_reproduction_help,
'fixed': fixed,
'fixed_full': fixed_full,
'issue_url': issue_url,
'is_admin': auth.is_current_user_admin(),
'metadata': metadata,
'minimized_testcase_size': minimized_testcase_size,
'needs_refresh': needs_refresh,
'original_testcase_size': original_testcase_size,
'privileged_user': privileged_user,
'regression': regression,
'crash_stacktrace': {
'lines':
crash_stacktrace,
'revision':
revisions.get_real_revision(crash_revision,
testcase.job_type,
display=True,
platform_id=testcase.platform_id)
},
'last_tested_crash_stacktrace': {
'lines':
last_tested_crash_stacktrace,
'revision':
revisions.get_real_revision(last_tested_crash_revision,
testcase.job_type,
display=True,
platform_id=testcase.platform_id)
},
'security_severity': security_severity,
'security_severities': data_types.SecuritySeverity.list(),
'stats': {
'min_hour': crash_stats.get_min_hour(),
'max_hour': crash_stats.get_max_hour(),
},
'suspected_cls': _parse_suspected_cls(metadata.get('predator_result')),
'testcase': testcase,
'timestamp': utils.utc_datetime_to_timestamp(testcase.timestamp),
'show_blame': testcase.has_blame(),
'show_impact': testcase.has_impacts(),
'impacts_production': testcase.impacts_production(),
'find_similar_issues_options': FIND_SIMILAR_ISSUES_OPTIONS,
'auto_delete_timestamp': auto_delete_timestamp,
'auto_close_timestamp': auto_close_timestamp,
'memory_tool_display_label': memory_tool_display_label,
'memory_tool_display_value': memory_tool_display_value,
'last_tested': last_tested,
'is_admin_or_not_oss_fuzz': is_admin_or_not_oss_fuzz(),
'has_issue_tracker': has_issue_tracker,
'reproduction_help_url': reproduction_help_url,
'is_local_development':
environment.is_running_on_app_engine_development(),
'fuzzer_display': fuzzer_display._asdict(),
}