class DataBundle(Model): """Represents a data bundle associated with a fuzzer.""" VALID_NAME_REGEX = NAME_CHECK_REGEX # The data bundle's name (important for identifying shared bundles). name = ndb.StringProperty() # Name of cloud storage bucket on GCS. bucket_name = ndb.StringProperty() # Data bundle's source (for accountability). # TODO(ochang): Remove. source = ndb.StringProperty() # If data bundle can be unpacked locally or needs nfs. is_local = ndb.BooleanProperty(default=True) # Creation timestamp. timestamp = ndb.DateTimeProperty() # Whether or not bundle should be synced to worker instead. # Fuzzer scripts are usually run on trusted hosts, so data bundles are synced # there. In libFuzzer's case, we want the bundle to be on the same machine as # where the libFuzzer binary will run (untrusted). sync_to_worker = ndb.BooleanProperty(default=False)
class TestcaseVariant(Model): """Represent a testcase variant on another job (another platform / sanitizer / config).""" # Testcase ID of the testcase for which the variant is being evaluated. testcase_id = ndb.IntegerProperty() # Status of the testcase variant (pending, reproducible, unreproducible, etc). status = ndb.IntegerProperty(default=0) # Job type for the testcase variant. job_type = ndb.StringProperty() # Revision that the testcase variant was tried against. revision = ndb.IntegerProperty() # Crash type. crash_type = ndb.StringProperty() # Crash state. crash_state = ndb.StringProperty() # Bool to indicate if it is a security bug? security_flag = ndb.BooleanProperty() # Bool to indicate if crash is similar to original testcase. is_similar = ndb.BooleanProperty() # Similar testcase reproducer key (optional). This is set in case we notice a # similar crash on another platform. reproducer_key = ndb.StringProperty()
class TestcaseUploadMetadata(Model): """Metadata associated with a user uploaded test case.""" # Timestamp. timestamp = ndb.DateTimeProperty() # Testcase filename. filename = StringProperty() # Current status of the testcase. status = StringProperty() # Uploader email address. uploader_email = StringProperty() # Name of the bot that ran analyze on this testcase. bot_name = StringProperty() # Id of the associated testcase. testcase_id = ndb.IntegerProperty() # Id of the testcase that this is marked as a duplicate of. duplicate_of = ndb.IntegerProperty() # Blobstore key for the testcase associated with this object. blobstore_key = StringProperty() # Testcase timeout. timeout = ndb.IntegerProperty() # Is this a single testcase bundled in an archive? bundled = ndb.BooleanProperty() # Path to the file in the archive. path_in_archive = TextProperty() # Original blobstore key for this object (used for archives). original_blobstore_key = StringProperty() # Security flag. security_flag = ndb.BooleanProperty(default=False) # Number of retries for this testcase. retries = ndb.IntegerProperty() # Flag to indicate where bug title should be updated or not. bug_summary_update_flag = ndb.BooleanProperty() # Flag to indicate if we are running in quiet mode (e.g. bug updates). quiet_flag = ndb.BooleanProperty() # Additional testcase metadata dict stored as a string. additional_metadata_string = TextProperty(indexed=False)
class FiledBug(Model): """Metadata information for issues that were filed automatically.""" # Timestamp when the issue was filed. timestamp = ndb.DateTimeProperty() # ID of the test case that is associated with the filed issue. testcase_id = ndb.IntegerProperty() # Tracking issue tracker bug for this testcase. bug_information = ndb.IntegerProperty(default=0) # Group ID associated with this issue. group_id = ndb.IntegerProperty() # Crash type for easy reference. crash_type = ndb.StringProperty() # Crash state for easy reference. crash_state = ndb.StringProperty() # Is it a security bug? security_flag = ndb.BooleanProperty() # Platform id. platform_id = ndb.StringProperty()
class ReportMetadata(Model): """Metadata associated with a crash report.""" # Job type from testcase. job_type = ndb.StringProperty() # Revision of build from report. crash_revision = ndb.IntegerProperty(default=-1) # Has this report been successfully uploaded? is_uploaded = ndb.BooleanProperty(default=False) # Product. product = ndb.StringProperty(default='') # Version. version = ndb.StringProperty(default='', indexed=False) # Key to minidump previously written to blobstore. minidump_key = ndb.StringProperty(default='', indexed=False) # Processed crash bytes. serialized_crash_stack_frames = ndb.BlobProperty(default='', indexed=False) # Id of the associated testcase. testcase_id = ndb.StringProperty(default='') # Id of the associated bot. bot_id = ndb.StringProperty(default='', indexed=False) # Optional upload params, stored as a JSON object. optional_params = ndb.TextProperty(indexed=False) # Report id from crash/. crash_report_id = ndb.StringProperty()
class BundledArchiveMetadata(Model): """Metadata needed for multiple test cases uploaded in an archive.""" # Blobstore key of the archive. blobstore_key = StringProperty() # Timeout in seconds for each testcase in the bundle. timeout = ndb.IntegerProperty() # Job queue for the analyze tasks created for this bundle. job_queue = StringProperty() # Job type that should be used for all testcases in this bundle. job_type = StringProperty() # Flag indicating whether or not these testcases need http. http_flag = ndb.BooleanProperty() # File name of the uploaded archive. archive_filename = StringProperty() # Email address of the uploader of the archive. uploader_email = StringProperty() # Fake user interaction sequences like key clicks, mouse movements, etc. gestures = StringProperty(repeated=True) # Optional. Revision that we discovered the crash in. crash_revision = ndb.IntegerProperty() # Optional. Additional arguments. additional_arguments = StringProperty() # Optional. Bug information. bug_information = StringProperty() # Optional. Platform id, e.g. android:shamu. platform_id = StringProperty() # Optional. App launch command. e.g. shell am start ... app_launch_command = StringProperty() # Fuzzer name. fuzzer_name = StringProperty() # Overridden fuzzer name because actual fuzzer name can be different in many # scenarios (libfuzzer, afl, etc). overridden_fuzzer_name = StringProperty() # Binary name for fuzz target (only applicable to libFuzzer, AFL). fuzzer_binary_name = StringProperty()
class ExternalUserPermission(Model): """Permissions for external users.""" # Email user is authenticated as. email = ndb.StringProperty() # Type of |entity_name|. Can be one of the values of PermissionEntityKind. entity_kind = ndb.IntegerProperty() # Name of the entity that user is allowed to view. entity_name = ndb.StringProperty() # Whether or not |allowed_name| is a prefix. is_prefix = ndb.BooleanProperty(default=False) # Auto CC type. auto_cc = ndb.IntegerProperty()
class OssFuzzProject(Model): """Represents a project that has been set up for OSS-Fuzz.""" # Name of the project. name = ndb.StringProperty() # Whether or not the project should run on high end hosts. high_end = ndb.BooleanProperty(default=False) # Weight for CPU distribution. This is set by admins. cpu_weight = ndb.FloatProperty(default=1.0) # The disk size to use (overrides the default). disk_size_gb = ndb.IntegerProperty() # Service account for this project. service_account = ndb.StringProperty() # CCs for the project. ccs = ndb.StringProperty(repeated=True)
class BuildMetadata(Model): """Metadata associated with a particular archived build.""" # Job type that this build belongs to. job_type = ndb.StringProperty() # Revision of the build. revision = ndb.IntegerProperty() # Good build or bad build. bad_build = ndb.BooleanProperty(default=False) # Stdout and stderr. console_output = ndb.TextProperty() # Bot name. bot_name = ndb.StringProperty() # Symbol data. symbols = ndb.StringProperty() # Creation timestamp. timestamp = ndb.DateTimeProperty()
class Config(Model): """Configuration.""" previous_hash = ndb.StringProperty(default='') # Project's url. url = ndb.StringProperty(default='') # Issue tracker client authentication parameters. client_credentials = ndb.TextProperty(default='') # Build apiary authentication parameters. build_apiary_service_account_email = ndb.StringProperty(default='') build_apiary_service_account_private_key = ndb.TextProperty(default='') # Google test account for login, gms testing, etc. test_account_email = ndb.StringProperty(default='') test_account_password = ndb.StringProperty(default='') # Privileged users. privileged_users = ndb.TextProperty(default='') # Admin contact string. contact_string = ndb.StringProperty(default='') # Component to repository mappings. component_repository_mappings = ndb.TextProperty(default='') # URL for help page for reproducing issues. reproduction_help_url = ndb.StringProperty(default='') # Documentation url. documentation_url = ndb.StringProperty(default='') # Bug report url. bug_report_url = ndb.StringProperty(default='') # Platforms that coverage is supported for. platform_group_mappings = ndb.TextProperty(default='') # More relaxed restrictions: allow CC'ed users and reporters of issues to view # testcase details. relax_testcase_restrictions = ndb.BooleanProperty(default=False) # More relaxed restrictions: allow domain users to access both security and # functional bugs. relax_security_bug_restrictions = ndb.BooleanProperty(default=False) # Coverage reports bucket. coverage_reports_bucket = ndb.StringProperty(default='') # For GitHub API. github_credentials = ndb.StringProperty(default='') # OAuth2 client id for the reproduce tool. reproduce_tool_client_id = ndb.StringProperty(default='') # OAuth2 client secret for the reproduce tool. reproduce_tool_client_secret = ndb.StringProperty(default='') # Pub/Sub topics for the Predator service. predator_crash_topic = ndb.StringProperty(default='') predator_result_topic = ndb.StringProperty(default='') # Wifi connection information. wifi_ssid = ndb.StringProperty(default='') wifi_password = ndb.StringProperty(default='') # SendGrid config. sendgrid_api_key = ndb.StringProperty(default='') sendgrid_sender = ndb.StringProperty(default='')
class Testcase(Model): """Represents a single testcase.""" # Crash on an invalid read/write. crash_type = ndb.StringProperty() # Crashing address. crash_address = ndb.StringProperty(indexed=False) # First x stack frames. crash_state = ndb.StringProperty() # Complete stacktrace. crash_stacktrace = ndb.TextProperty(indexed=False) # Last tested crash stacktrace using the latest revision. last_tested_crash_stacktrace = ndb.TextProperty(indexed=False) # Blobstore keys for various things like original testcase, minimized # testcase, etc. fuzzed_keys = ndb.StringProperty(indexed=False) minimized_keys = ndb.StringProperty(indexed=False) minidump_keys = ndb.StringProperty(indexed=False) # Tracking issue tracker bug. One bug number per line (future extension). bug_information = ndb.StringProperty() # Regression range. regression = ndb.StringProperty(default='') # Revisions where this issue has been fixed. fixed = ndb.StringProperty(default='') # Is it a security bug ? security_flag = ndb.BooleanProperty(default=False) # Security severity of the bug. security_severity = ndb.IntegerProperty(indexed=False) # Did the bug only reproduced once ? one_time_crasher_flag = ndb.BooleanProperty(default=False) # Any additional comments. comments = ndb.TextProperty(default='', indexed=False) # Revision that we discovered the crash in. crash_revision = ndb.IntegerProperty() # The file on the bot that generated the testcase. original_absolute_path = ndb.StringProperty(indexed=False, default='') absolute_path = ndb.StringProperty(indexed=False) # Minimized argument list. minimized_arguments = ndb.TextProperty(default='', indexed=False) # Window argument (usually width, height, top, left, etc). window_argument = ndb.TextProperty(default='', indexed=False) # Type of job associated with this testcase. job_type = ndb.StringProperty() # Original job queue used for tasks created for this testcase. queue = ndb.StringProperty(indexed=False) # State representing whether the fuzzed or minimized testcases are archived. archive_state = ndb.IntegerProperty(default=0, indexed=False) # File name of the original uploaded archive. archive_filename = ndb.StringProperty(indexed=False) # Is this a binary file? binary_flag = ndb.BooleanProperty(default=False, indexed=False) # Timestamp. timestamp = ndb.DateTimeProperty() # Does the testcase crash stack vary b/w crashes ? flaky_stack = ndb.BooleanProperty(default=False, indexed=False) # Do we need to test this testcase using an HTTP/HTTPS server? http_flag = ndb.BooleanProperty(default=False, indexed=False) # Name of the fuzzer used to generate this testcase. fuzzer_name = ndb.StringProperty() # Status of this testcase (pending, processed, unreproducible, etc). status = ndb.StringProperty(default='Processed') # Id of the testcase that this is marked as a duplicate of. duplicate_of = ndb.IntegerProperty(indexed=False) # Flag indicating whether or not the testcase has been symbolized. symbolized = ndb.BooleanProperty(default=False, indexed=False) # Id for this testcase's associated group. group_id = ndb.IntegerProperty(default=0) # Tracking issue tracker bug for this testcase group. group_bug_information = ndb.IntegerProperty(default=0) # Fake user interaction sequences like key clicks, mouse movements, etc. gestures = ndb.StringProperty(repeated=True, indexed=False) # ASAN redzone size in bytes. redzone = ndb.IntegerProperty(default=128, indexed=False) # Whether testcase is open. open = ndb.BooleanProperty(default=True) # Adjusts timeout based on multiplier value. timeout_multiplier = ndb.FloatProperty(default=1.0, indexed=False) # Additional metadata stored as a JSON object. This should be used for # properties that are not commonly accessed and do not need to be indexed. additional_metadata = ndb.TextProperty(indexed=False) # Boolean attribute indicating if cleanup triage needs to be done. triaged = ndb.BooleanProperty(default=False) # Project name associated with this test case. project_name = ndb.StringProperty() # keywords is used for searching. keywords = ndb.StringProperty(repeated=True) # Whether testcase has a bug (either bug_information or # group_bug_information). has_bug_flag = ndb.BooleanProperty() # Indices for bug_information and group_bug_information. bug_indices = ndb.StringProperty(repeated=True) # Overridden fuzzer name because actual fuzzer name can be different in many # scenarios (libfuzzer, afl, etc). overridden_fuzzer_name = ndb.StringProperty() # Platform (e.g. windows, linux, android). platform = ndb.StringProperty() # Platform id (e.g. windows, linux, android:hammerhead:l). # For Android, includes device type and underlying OS version. platform_id = ndb.StringProperty() # Impact indices for searching. impact_indices = ndb.StringProperty(repeated=True) # Whether or not a testcase is a duplicate of other testcase. is_a_duplicate_flag = ndb.BooleanProperty() # Whether or not a testcase is the leader of its group. # If the testcase is not in a group, it's the leader of a group of 1. # The default is false because we prefer not to show crashes until we are # sure. And group_task will correctly set the value within 30 minutes. is_leader = ndb.BooleanProperty(default=False) # Fuzzer name indices fuzzer_name_indices = ndb.StringProperty(repeated=True) # The impacted version indices (including both beta and stable). impact_version_indices = ndb.StringProperty(repeated=True) # The impacted stable version. impact_stable_version = ndb.StringProperty() # The impacted stable version indices. impact_stable_version_indices = ndb.StringProperty(repeated=True) # The impacted stable version is merely probable (not definite). Because # for a non-asan build, we don't have a stable/beta build. Therefore, we # make an intelligent guess on the version. impact_stable_version_likely = ndb.BooleanProperty() # The impacted beta version. impact_beta_version = ndb.StringProperty() # The impacted beta version indices. impact_beta_version_indices = ndb.StringProperty(repeated=True) # The impacted beta version is merely probable (not definite). See the # comment on impact_stable_version_likely. impact_beta_version_likely = ndb.BooleanProperty() # Whether or not impact task has been run on this testcase. is_impact_set_flag = ndb.BooleanProperty() # Code coverage data for the testcase. coverage = ndb.StringProperty() # Uploader email address. uploader_email = ndb.StringProperty() def has_blame(self): return self.project_name == 'chromium' def has_impacts(self): return self.project_name == 'chromium' and not self.one_time_crasher_flag def impacts_production(self): return bool(self.impact_stable_version) or bool( self.impact_beta_version) def is_status_unreproducible(self): return self.status and self.status.startswith('Unreproducible') def is_crash(self): return bool(self.crash_state) def populate_indices(self): """Populate keywords for fast test case list searching.""" self.keywords = list( search_tokenizer.tokenize(self.crash_state) | search_tokenizer.tokenize(self.crash_type) | search_tokenizer.tokenize(self.fuzzer_name) | search_tokenizer.tokenize(self.overridden_fuzzer_name) | search_tokenizer.tokenize(self.job_type) | search_tokenizer.tokenize(self.platform_id)) self.bug_indices = search_tokenizer.tokenize_bug_information(self) self.has_bug_flag = bool(self.bug_indices) self.is_a_duplicate_flag = bool(self.duplicate_of) fuzzer_name_indices = list( set([self.fuzzer_name, self.overridden_fuzzer_name])) self.fuzzer_name_indices = [f for f in fuzzer_name_indices if f] # If the impact task hasn't been run (aka is_impact_set_flag=False) OR # if impact isn't applicable (aka has_impacts() is False), we wipe all # the impact fields' indices. if self.has_impacts() and self.is_impact_set_flag: self.impact_stable_version_indices = ( search_tokenizer.tokenize_impact_version( self.impact_stable_version)) self.impact_beta_version_indices = ( search_tokenizer.tokenize_impact_version( self.impact_beta_version)) self.impact_version_indices = list( set(self.impact_stable_version_indices + self.impact_beta_version_indices)) if self.impact_beta_version: self.impact_version_indices.append('beta') if self.impact_stable_version: self.impact_version_indices.append('stable') if not self.impacts_production(): self.impact_version_indices.append('head') else: self.impact_version_indices = [] self.impact_stable_version_indices = [] self.impact_beta_version_indices = [] def _pre_put_hook(self): self.populate_indices() def _post_put_hook(self, _): logs.log('Updated testcase %d (bug %s).' % (self.key.id(), self.bug_information or '-')) def set_impacts_as_na(self): self.impact_stable_version = self.impact_beta_version = None self.impact_stable_version_likely = self.impact_beta_version_likely = False self.is_impact_set_flag = False def _ensure_metadata_is_cached(self): """Ensure that the metadata for this has been cached.""" if hasattr(self, 'metadata_cache'): return try: cache = json_utils.loads(self.additional_metadata) except (TypeError, ValueError): cache = {} setattr(self, 'metadata_cache', cache) def get_metadata(self, key=None, default=None): """Get metadata for a test case. Slow on first access.""" self._ensure_metadata_is_cached() # If no key is specified, return all metadata. if not key: return self.metadata_cache try: return self.metadata_cache[key] except KeyError: return default def set_metadata(self, key, value, update_testcase=True): """Set metadata for a test case.""" self._ensure_metadata_is_cached() self.metadata_cache[key] = value self.additional_metadata = json_utils.dumps(self.metadata_cache) if update_testcase: self.put() def delete_metadata(self, key, update_testcase=True): """Remove metadata key for a test case.""" self._ensure_metadata_is_cached() # Make sure that the key exists in cache. If not, no work to do here. if key not in self.metadata_cache: return del self.metadata_cache[key] self.additional_metadata = json_utils.dumps(self.metadata_cache) if update_testcase: self.put() def actual_fuzzer_name(self): """Actual fuzzer name, uses one from overridden attribute if available.""" return self.overridden_fuzzer_name or self.fuzzer_name def get_fuzz_target(self): """Get the associated FuzzTarget entity for this test case.""" name = self.actual_fuzzer_name() if not name: return None return ndb.Key(FuzzTarget, name).get()
class Fuzzer(Model): """Represents a fuzzer.""" VALID_NAME_REGEX = NAME_CHECK_REGEX # Last update time. timestamp = ndb.DateTimeProperty() # Fuzzer Name. name = ndb.StringProperty() # The name of the archive that the user uploaded. filename = ndb.StringProperty() # Blobstore key for this fuzzer. blobstore_key = ndb.StringProperty() # String representation of the file size. file_size = ndb.StringProperty() # Fuzzer's main executable path, relative to root. executable_path = ndb.StringProperty() # Revision number of the fuzzer. revision = ndb.IntegerProperty() # Fuzzer's source (for accountability). source = ndb.StringProperty() # Testcase timeout. timeout = ndb.IntegerProperty() # Supported platforms. supported_platforms = ndb.StringProperty() # Custom script that should be used to launch chrome for this fuzzer. launcher_script = ndb.StringProperty() # Result from the last fuzzer run showing the number of testcases generated. result = ndb.StringProperty() # Last result update timestamp. result_timestamp = ndb.DateTimeProperty() # Console output from last fuzzer run. console_output = ndb.TextProperty() # Return code from last fuzzer run. return_code = ndb.IntegerProperty() # Blobstore key for the sample testcase generated by the fuzzer. sample_testcase = ndb.StringProperty() # Job types for this fuzzer. jobs = ndb.StringProperty(repeated=True) # Is the fuzzer coming from an external contributor ? Useful for adding # reward flags. external_contribution = ndb.BooleanProperty(default=False) # Max testcases to generate for this fuzzer. max_testcases = ndb.IntegerProperty() # Does it run un-trusted content ? Examples including running live sites. untrusted_content = ndb.BooleanProperty(default=False) # Data bundle name. data_bundle_name = ndb.StringProperty(default='') # Additional environment variables that need to be set for this fuzzer. additional_environment_string = ndb.TextProperty() # Column specification for stats. stats_columns = ndb.StringProperty(indexed=False) # Helpful descriptions for the stats_columns. In a yaml format. stats_column_descriptions = ndb.TextProperty(indexed=False) # Whether this is a builtin fuzzer. builtin = ndb.BooleanProperty(indexed=False, default=False) # Whether this is a differential fuzzer. differential = ndb.BooleanProperty(default=False)
class Config(Model): """Configuration.""" previous_hash = ndb.StringProperty(default='') # Project's url. url = ndb.StringProperty(default='') # Issue tracker client authentication parameters. client_credentials = ndb.TextProperty(default='') # Build apiary authentication parameters. build_apiary_service_account_email = ndb.StringProperty(default='') build_apiary_service_account_private_key = ndb.TextProperty(default='') # Google test account for login, gms testing, etc. test_account_email = ndb.StringProperty(default='') test_account_password = ndb.StringProperty(default='') # Privileged users. privileged_users = ndb.TextProperty(default='') # Stack blacklist/exclusions. stack_blacklist = ndb.TextProperty(default='') # Stacktrace clean regexes. stack_clean_regex = ndb.TextProperty(default='') # Admin contact string. contact_string = ndb.StringProperty(default='') # Revision variables url (diff). revision_vars_url = ndb.TextProperty(default='') # Component to repository mappings. component_repository_mappings = ndb.TextProperty(default='') # URL for help page for reproducing issues. reproduction_help_url = ndb.StringProperty(default='') # Documentation url. documentation_url = ndb.StringProperty(default='') # Bug report url. bug_report_url = ndb.StringProperty(default='') # Platforms that coverage is supported for. platform_group_mappings = ndb.TextProperty(default='') # More relaxed restrictions: allow CC'ed users and reporters of issues to view # testcase details. relax_testcase_restrictions = ndb.BooleanProperty(default=False) # Coverage reports bucket. coverage_reports_bucket = ndb.StringProperty(default='') # For GitHub API. github_credentials = ndb.StringProperty(default='') # OAuth2 client secret for ClusterFuzz tools. clusterfuzz_tools_client_secret = ndb.StringProperty(default='') # Pub/Sub topics for the Predator service. predator_crash_topic = ndb.StringProperty(default='') predator_result_topic = ndb.StringProperty(default='') # Wifi connection information. wifi_ssid = ndb.StringProperty(default='') wifi_password = ndb.StringProperty(default='')
class TestDatastoreModel(ndb.Model): tokens = ndb.StringProperty(repeated=True) boolean_value = ndb.BooleanProperty() datetime_value = ndb.DateTimeProperty()