class ReportMetadata(Model):
"""Metadata associated with a crash report."""
# Job type from testcase.
job_type = ndb.StringProperty()
# Revision of build from report.
crash_revision = ndb.IntegerProperty(default=-1)
# Has this report been successfully uploaded?
is_uploaded = ndb.BooleanProperty(default=False)
# Product.
product = ndb.StringProperty(default='')
# Version.
version = ndb.StringProperty(default='', indexed=False)
# Key to minidump previously written to blobstore.
minidump_key = ndb.StringProperty(default='', indexed=False)
# Processed crash bytes.
serialized_crash_stack_frames = ndb.BlobProperty(default='', indexed=False)
# Id of the associated testcase.
testcase_id = ndb.StringProperty(default='')
# Id of the associated bot.
bot_id = ndb.StringProperty(default='', indexed=False)
# Optional upload params, stored as a JSON object.
optional_params = ndb.TextProperty(indexed=False)
# Report id from crash/.
crash_report_id = ndb.StringProperty()
class TestcaseUploadMetadata(Model):
"""Metadata associated with a user uploaded test case."""
# Timestamp.
timestamp = ndb.DateTimeProperty()
# Testcase filename.
filename = ndb.StringProperty()
# Current status of the testcase.
status = ndb.StringProperty()
# Uploader email address.
uploader_email = ndb.StringProperty()
# Name of the bot that ran analyze on this testcase.
bot_name = ndb.StringProperty()
# Id of the associated testcase.
testcase_id = ndb.IntegerProperty()
# Id of the testcase that this is marked as a duplicate of.
duplicate_of = ndb.IntegerProperty()
# Blobstore key for the testcase associated with this object.
blobstore_key = ndb.StringProperty()
# Testcase timeout.
timeout = ndb.IntegerProperty()
# Is this a single testcase bundled in an archive?
bundled = ndb.BooleanProperty()
# Path to the file in the archive.
path_in_archive = ndb.TextProperty()
# Original blobstore key for this object (used for archives).
original_blobstore_key = ndb.StringProperty()
# Security flag.
security_flag = ndb.BooleanProperty(default=False)
# Number of retries for this testcase.
retries = ndb.IntegerProperty()
# Flag to indicate where bug title should be updated or not.
bug_summary_update_flag = ndb.BooleanProperty()
# Flag to indicate if we are running in quiet mode (e.g. bug updates).
quiet_flag = ndb.BooleanProperty()
class BuildMetadata(Model):
"""Metadata associated with a particular archived build."""
# Job type that this build belongs to.
job_type = ndb.StringProperty()
# Revision of the build.
revision = ndb.IntegerProperty()
# Good build or bad build.
bad_build = ndb.BooleanProperty(default=False)
# Stdout and stderr.
console_output = ndb.TextProperty()
# Bot name.
bot_name = ndb.StringProperty()
# Symbol data.
symbols = ndb.StringProperty()
# Creation timestamp.
timestamp = ndb.DateTimeProperty()
class Job(Model):
"""Definition of a job type used by the bots."""
VALID_NAME_REGEX = NAME_CHECK_REGEX
# Job type name.
name = ndb.StringProperty()
# Job environment string.
environment_string = ndb.TextProperty()
# The platform that this job can run on.
platform = ndb.StringProperty()
# Blobstore key of the custom binary for this job.
custom_binary_key = ndb.StringProperty()
# Filename for the custom binary.
custom_binary_filename = ndb.StringProperty()
# Revision of the custom binary.
custom_binary_revision = ndb.IntegerProperty()
# Description of the job.
description = ndb.TextProperty()
# Template to use, if any.
templates = ndb.StringProperty(repeated=True)
def get_environment(self):
"""Get the environment as a dict for this job, including any environment
variables in its template."""
if not self.templates:
return environment.parse_environment_definition(
self.environment_string)
job_environment = {}
for template_name in self.templates:
template = JobTemplate.query(
JobTemplate.name == template_name).get()
if not template:
continue
template_environment = environment.parse_environment_definition(
template.environment_string)
job_environment.update(template_environment)
environment_overrides = environment.parse_environment_definition(
self.environment_string)
job_environment.update(environment_overrides)
return job_environment
def get_environment_string(self):
"""Get the environment string for this job, including any environment
variables in its template. Avoid using this if possible."""
environment_string = ''
job_environment = self.get_environment()
for key, value in six.iteritems(job_environment):
environment_string += '%s = %s\n' % (key, value)
return environment_string
class JobTemplate(Model):
# Job template name.
name = ndb.StringProperty()
# Environment string.
environment_string = ndb.TextProperty()
class Config(Model):
"""Configuration."""
previous_hash = ndb.StringProperty(default='')
# Project's url.
url = ndb.StringProperty(default='')
# Issue tracker client authentication parameters.
client_credentials = ndb.TextProperty(default='')
# Build apiary authentication parameters.
build_apiary_service_account_email = ndb.StringProperty(default='')
build_apiary_service_account_private_key = ndb.TextProperty(default='')
# Google test account for login, gms testing, etc.
test_account_email = ndb.StringProperty(default='')
test_account_password = ndb.StringProperty(default='')
# Privileged users.
privileged_users = ndb.TextProperty(default='')
# Admin contact string.
contact_string = ndb.StringProperty(default='')
# Component to repository mappings.
component_repository_mappings = ndb.TextProperty(default='')
# URL for help page for reproducing issues.
reproduction_help_url = ndb.StringProperty(default='')
# Documentation url.
documentation_url = ndb.StringProperty(default='')
# Bug report url.
bug_report_url = ndb.StringProperty(default='')
# Platforms that coverage is supported for.
platform_group_mappings = ndb.TextProperty(default='')
# More relaxed restrictions: allow CC'ed users and reporters of issues to view
# testcase details.
relax_testcase_restrictions = ndb.BooleanProperty(default=False)
# More relaxed restrictions: allow domain users to access both security and
# functional bugs.
relax_security_bug_restrictions = ndb.BooleanProperty(default=False)
# Coverage reports bucket.
coverage_reports_bucket = ndb.StringProperty(default='')
# For GitHub API.
github_credentials = ndb.StringProperty(default='')
# OAuth2 client id for the reproduce tool.
reproduce_tool_client_id = ndb.StringProperty(default='')
# OAuth2 client secret for the reproduce tool.
reproduce_tool_client_secret = ndb.StringProperty(default='')
# Pub/Sub topics for the Predator service.
predator_crash_topic = ndb.StringProperty(default='')
predator_result_topic = ndb.StringProperty(default='')
# Wifi connection information.
wifi_ssid = ndb.StringProperty(default='')
wifi_password = ndb.StringProperty(default='')
# SendGrid config.
sendgrid_api_key = ndb.StringProperty(default='')
sendgrid_sender = ndb.StringProperty(default='')
class Testcase(Model):
"""Represents a single testcase."""
# Crash on an invalid read/write.
crash_type = ndb.StringProperty()
# Crashing address.
crash_address = ndb.StringProperty(indexed=False)
# First x stack frames.
crash_state = ndb.StringProperty()
# Complete stacktrace.
crash_stacktrace = ndb.TextProperty(indexed=False)
# Last tested crash stacktrace using the latest revision.
last_tested_crash_stacktrace = ndb.TextProperty(indexed=False)
# Blobstore keys for various things like original testcase, minimized
# testcase, etc.
fuzzed_keys = ndb.StringProperty(indexed=False)
minimized_keys = ndb.StringProperty(indexed=False)
minidump_keys = ndb.StringProperty(indexed=False)
# Tracking issue tracker bug. One bug number per line (future extension).
bug_information = ndb.StringProperty()
# Regression range.
regression = ndb.StringProperty(default='')
# Revisions where this issue has been fixed.
fixed = ndb.StringProperty(default='')
# Is it a security bug ?
security_flag = ndb.BooleanProperty(default=False)
# Security severity of the bug.
security_severity = ndb.IntegerProperty(indexed=False)
# Did the bug only reproduced once ?
one_time_crasher_flag = ndb.BooleanProperty(default=False)
# Any additional comments.
comments = ndb.TextProperty(default='', indexed=False)
# Revision that we discovered the crash in.
crash_revision = ndb.IntegerProperty()
# The file on the bot that generated the testcase.
original_absolute_path = ndb.StringProperty(indexed=False, default='')
absolute_path = ndb.StringProperty(indexed=False)
# Minimized argument list.
minimized_arguments = ndb.TextProperty(default='', indexed=False)
# Window argument (usually width, height, top, left, etc).
window_argument = ndb.TextProperty(default='', indexed=False)
# Type of job associated with this testcase.
job_type = ndb.StringProperty()
# Original job queue used for tasks created for this testcase.
queue = ndb.StringProperty(indexed=False)
# State representing whether the fuzzed or minimized testcases are archived.
archive_state = ndb.IntegerProperty(default=0, indexed=False)
# File name of the original uploaded archive.
archive_filename = ndb.StringProperty(indexed=False)
# Is this a binary file?
binary_flag = ndb.BooleanProperty(default=False, indexed=False)
# Timestamp.
timestamp = ndb.DateTimeProperty()
# Does the testcase crash stack vary b/w crashes ?
flaky_stack = ndb.BooleanProperty(default=False, indexed=False)
# Do we need to test this testcase using an HTTP/HTTPS server?
http_flag = ndb.BooleanProperty(default=False, indexed=False)
# Name of the fuzzer used to generate this testcase.
fuzzer_name = ndb.StringProperty()
# Status of this testcase (pending, processed, unreproducible, etc).
status = ndb.StringProperty(default='Processed')
# Id of the testcase that this is marked as a duplicate of.
duplicate_of = ndb.IntegerProperty(indexed=False)
# Flag indicating whether or not the testcase has been symbolized.
symbolized = ndb.BooleanProperty(default=False, indexed=False)
# Id for this testcase's associated group.
group_id = ndb.IntegerProperty(default=0)
# Tracking issue tracker bug for this testcase group.
group_bug_information = ndb.IntegerProperty(default=0)
# Fake user interaction sequences like key clicks, mouse movements, etc.
gestures = ndb.StringProperty(repeated=True, indexed=False)
# ASAN redzone size in bytes.
redzone = ndb.IntegerProperty(default=128, indexed=False)
# Whether testcase is open.
open = ndb.BooleanProperty(default=True)
# Adjusts timeout based on multiplier value.
timeout_multiplier = ndb.FloatProperty(default=1.0, indexed=False)
# Additional metadata stored as a JSON object. This should be used for
# properties that are not commonly accessed and do not need to be indexed.
additional_metadata = ndb.TextProperty(indexed=False)
# Boolean attribute indicating if cleanup triage needs to be done.
triaged = ndb.BooleanProperty(default=False)
# Project name associated with this test case.
project_name = ndb.StringProperty()
# keywords is used for searching.
keywords = ndb.StringProperty(repeated=True)
# Whether testcase has a bug (either bug_information or
# group_bug_information).
has_bug_flag = ndb.BooleanProperty()
# Indices for bug_information and group_bug_information.
bug_indices = ndb.StringProperty(repeated=True)
# Overridden fuzzer name because actual fuzzer name can be different in many
# scenarios (libfuzzer, afl, etc).
overridden_fuzzer_name = ndb.StringProperty()
# Platform (e.g. windows, linux, android).
platform = ndb.StringProperty()
# Platform id (e.g. windows, linux, android:hammerhead:l).
# For Android, includes device type and underlying OS version.
platform_id = ndb.StringProperty()
# Impact indices for searching.
impact_indices = ndb.StringProperty(repeated=True)
# Whether or not a testcase is a duplicate of other testcase.
is_a_duplicate_flag = ndb.BooleanProperty()
# Whether or not a testcase is the leader of its group.
# If the testcase is not in a group, it's the leader of a group of 1.
# The default is false because we prefer not to show crashes until we are
# sure. And group_task will correctly set the value within 30 minutes.
is_leader = ndb.BooleanProperty(default=False)
# Fuzzer name indices
fuzzer_name_indices = ndb.StringProperty(repeated=True)
# The impacted version indices (including both beta and stable).
impact_version_indices = ndb.StringProperty(repeated=True)
# The impacted stable version.
impact_stable_version = ndb.StringProperty()
# The impacted stable version indices.
impact_stable_version_indices = ndb.StringProperty(repeated=True)
# The impacted stable version is merely probable (not definite). Because
# for a non-asan build, we don't have a stable/beta build. Therefore, we
# make an intelligent guess on the version.
impact_stable_version_likely = ndb.BooleanProperty()
# The impacted beta version.
impact_beta_version = ndb.StringProperty()
# The impacted beta version indices.
impact_beta_version_indices = ndb.StringProperty(repeated=True)
# The impacted beta version is merely probable (not definite). See the
# comment on impact_stable_version_likely.
impact_beta_version_likely = ndb.BooleanProperty()
# Whether or not impact task has been run on this testcase.
is_impact_set_flag = ndb.BooleanProperty()
# Code coverage data for the testcase.
coverage = ndb.StringProperty()
# Uploader email address.
uploader_email = ndb.StringProperty()
def has_blame(self):
return self.project_name == 'chromium'
def has_impacts(self):
return self.project_name == 'chromium' and not self.one_time_crasher_flag
def impacts_production(self):
return bool(self.impact_stable_version) or bool(
self.impact_beta_version)
def is_status_unreproducible(self):
return self.status and self.status.startswith('Unreproducible')
def is_crash(self):
return bool(self.crash_state)
def populate_indices(self):
"""Populate keywords for fast test case list searching."""
self.keywords = list(
search_tokenizer.tokenize(self.crash_state)
| search_tokenizer.tokenize(self.crash_type)
| search_tokenizer.tokenize(self.fuzzer_name)
| search_tokenizer.tokenize(self.overridden_fuzzer_name)
| search_tokenizer.tokenize(self.job_type)
| search_tokenizer.tokenize(self.platform_id))
self.bug_indices = search_tokenizer.tokenize_bug_information(self)
self.has_bug_flag = bool(self.bug_indices)
self.is_a_duplicate_flag = bool(self.duplicate_of)
fuzzer_name_indices = list(
set([self.fuzzer_name, self.overridden_fuzzer_name]))
self.fuzzer_name_indices = [f for f in fuzzer_name_indices if f]
# If the impact task hasn't been run (aka is_impact_set_flag=False) OR
# if impact isn't applicable (aka has_impacts() is False), we wipe all
# the impact fields' indices.
if self.has_impacts() and self.is_impact_set_flag:
self.impact_stable_version_indices = (
search_tokenizer.tokenize_impact_version(
self.impact_stable_version))
self.impact_beta_version_indices = (
search_tokenizer.tokenize_impact_version(
self.impact_beta_version))
self.impact_version_indices = list(
set(self.impact_stable_version_indices +
self.impact_beta_version_indices))
if self.impact_beta_version:
self.impact_version_indices.append('beta')
if self.impact_stable_version:
self.impact_version_indices.append('stable')
if not self.impacts_production():
self.impact_version_indices.append('head')
else:
self.impact_version_indices = []
self.impact_stable_version_indices = []
self.impact_beta_version_indices = []
def _pre_put_hook(self):
self.populate_indices()
def _post_put_hook(self, _):
logs.log('Updated testcase %d (bug %s).' %
(self.key.id(), self.bug_information or '-'))
def set_impacts_as_na(self):
self.impact_stable_version = self.impact_beta_version = None
self.impact_stable_version_likely = self.impact_beta_version_likely = False
self.is_impact_set_flag = False
def _ensure_metadata_is_cached(self):
"""Ensure that the metadata for this has been cached."""
if hasattr(self, 'metadata_cache'):
return
try:
cache = json_utils.loads(self.additional_metadata)
except (TypeError, ValueError):
cache = {}
setattr(self, 'metadata_cache', cache)
def get_metadata(self, key=None, default=None):
"""Get metadata for a test case. Slow on first access."""
self._ensure_metadata_is_cached()
# If no key is specified, return all metadata.
if not key:
return self.metadata_cache
try:
return self.metadata_cache[key]
except KeyError:
return default
def set_metadata(self, key, value, update_testcase=True):
"""Set metadata for a test case."""
self._ensure_metadata_is_cached()
self.metadata_cache[key] = value
self.additional_metadata = json_utils.dumps(self.metadata_cache)
if update_testcase:
self.put()
def delete_metadata(self, key, update_testcase=True):
"""Remove metadata key for a test case."""
self._ensure_metadata_is_cached()
# Make sure that the key exists in cache. If not, no work to do here.
if key not in self.metadata_cache:
return
del self.metadata_cache[key]
self.additional_metadata = json_utils.dumps(self.metadata_cache)
if update_testcase:
self.put()
def actual_fuzzer_name(self):
"""Actual fuzzer name, uses one from overridden attribute if available."""
return self.overridden_fuzzer_name or self.fuzzer_name
def get_fuzz_target(self):
"""Get the associated FuzzTarget entity for this test case."""
name = self.actual_fuzzer_name()
if not name:
return None
return ndb.Key(FuzzTarget, name).get()
class Fuzzer(Model):
"""Represents a fuzzer."""
VALID_NAME_REGEX = NAME_CHECK_REGEX
# Last update time.
timestamp = ndb.DateTimeProperty()
# Fuzzer Name.
name = ndb.StringProperty()
# The name of the archive that the user uploaded.
filename = ndb.StringProperty()
# Blobstore key for this fuzzer.
blobstore_key = ndb.StringProperty()
# String representation of the file size.
file_size = ndb.StringProperty()
# Fuzzer's main executable path, relative to root.
executable_path = ndb.StringProperty()
# Revision number of the fuzzer.
revision = ndb.IntegerProperty()
# Fuzzer's source (for accountability).
source = ndb.StringProperty()
# Testcase timeout.
timeout = ndb.IntegerProperty()
# Supported platforms.
supported_platforms = ndb.StringProperty()
# Custom script that should be used to launch chrome for this fuzzer.
launcher_script = ndb.StringProperty()
# Result from the last fuzzer run showing the number of testcases generated.
result = ndb.StringProperty()
# Last result update timestamp.
result_timestamp = ndb.DateTimeProperty()
# Console output from last fuzzer run.
console_output = ndb.TextProperty()
# Return code from last fuzzer run.
return_code = ndb.IntegerProperty()
# Blobstore key for the sample testcase generated by the fuzzer.
sample_testcase = ndb.StringProperty()
# Job types for this fuzzer.
jobs = ndb.StringProperty(repeated=True)
# Is the fuzzer coming from an external contributor ? Useful for adding
# reward flags.
external_contribution = ndb.BooleanProperty(default=False)
# Max testcases to generate for this fuzzer.
max_testcases = ndb.IntegerProperty()
# Does it run un-trusted content ? Examples including running live sites.
untrusted_content = ndb.BooleanProperty(default=False)
# Data bundle name.
data_bundle_name = ndb.StringProperty(default='')
# Additional environment variables that need to be set for this fuzzer.
additional_environment_string = ndb.TextProperty()
# Column specification for stats.
stats_columns = ndb.StringProperty(indexed=False)
# Helpful descriptions for the stats_columns. In a yaml format.
stats_column_descriptions = ndb.TextProperty(indexed=False)
# Whether this is a builtin fuzzer.
builtin = ndb.BooleanProperty(indexed=False, default=False)
# Whether this is a differential fuzzer.
differential = ndb.BooleanProperty(default=False)
class Config(Model):
"""Configuration."""
previous_hash = ndb.StringProperty(default='')
# Project's url.
url = ndb.StringProperty(default='')
# Issue tracker client authentication parameters.
client_credentials = ndb.TextProperty(default='')
# Build apiary authentication parameters.
build_apiary_service_account_email = ndb.StringProperty(default='')
build_apiary_service_account_private_key = ndb.TextProperty(default='')
# Google test account for login, gms testing, etc.
test_account_email = ndb.StringProperty(default='')
test_account_password = ndb.StringProperty(default='')
# Privileged users.
privileged_users = ndb.TextProperty(default='')
# Stack blacklist/exclusions.
stack_blacklist = ndb.TextProperty(default='')
# Stacktrace clean regexes.
stack_clean_regex = ndb.TextProperty(default='')
# Admin contact string.
contact_string = ndb.StringProperty(default='')
# Revision variables url (diff).
revision_vars_url = ndb.TextProperty(default='')
# Component to repository mappings.
component_repository_mappings = ndb.TextProperty(default='')
# URL for help page for reproducing issues.
reproduction_help_url = ndb.StringProperty(default='')
# Documentation url.
documentation_url = ndb.StringProperty(default='')
# Bug report url.
bug_report_url = ndb.StringProperty(default='')
# Platforms that coverage is supported for.
platform_group_mappings = ndb.TextProperty(default='')
# More relaxed restrictions: allow CC'ed users and reporters of issues to view
# testcase details.
relax_testcase_restrictions = ndb.BooleanProperty(default=False)
# Coverage reports bucket.
coverage_reports_bucket = ndb.StringProperty(default='')
# For GitHub API.
github_credentials = ndb.StringProperty(default='')
# OAuth2 client secret for ClusterFuzz tools.
clusterfuzz_tools_client_secret = ndb.StringProperty(default='')
# Pub/Sub topics for the Predator service.
predator_crash_topic = ndb.StringProperty(default='')
predator_result_topic = ndb.StringProperty(default='')
# Wifi connection information.
wifi_ssid = ndb.StringProperty(default='')
wifi_password = ndb.StringProperty(default='')