def execute_task(*_): """Execute the report uploads.""" logs.log('Uploading pending reports.') # Get metadata for reports requiring upload. reports_metadata = ndb_utils.get_all_from_query( data_types.ReportMetadata.query( ndb_utils.is_false(data_types.ReportMetadata.is_uploaded))) reports_metadata = list(reports_metadata) if not reports_metadata: logs.log('No reports that need upload found.') return environment.set_value('UPLOAD_MODE', 'prod') # Otherwise, upload corresponding reports. logs.log('Uploading reports for testcases: %s' % str([report.testcase_id for report in reports_metadata])) report_metadata_to_delete = [] for report_metadata in reports_metadata: # Convert metadata back into actual report. crash_info = crash_uploader.crash_report_info_from_metadata( report_metadata) testcase_id = report_metadata.testcase_id try: _ = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: logs.log_warn('Could not find testcase %s.' % testcase_id) report_metadata_to_delete.append(report_metadata.key) continue # Upload the report and update the corresponding testcase info. logs.log('Processing testcase %s for crash upload.' % testcase_id) crash_report_id = crash_info.upload() if crash_report_id is None: logs.log_error( 'Crash upload for testcase %s failed, retry later.' % testcase_id) continue # Update the report metadata to indicate successful upload. report_metadata.crash_report_id = crash_report_id report_metadata.is_uploaded = True report_metadata.put() logs.log('Uploaded testcase %s to crash, got back report id %s.' % (testcase_id, crash_report_id)) time.sleep(1) # Delete report metadata entries where testcase does not exist anymore or # upload is not supported. if report_metadata_to_delete: ndb_utils.delete_multi(report_metadata_to_delete) # Log done with uploads. # Deletion happens in batches in cleanup_task, so that in case of error there # is some buffer for looking at stored ReportMetadata in the meantime. logs.log('Finished uploading crash reports.')
def _allowed_users_for_entity(name, entity_kind, auto_cc=None): """Return a list of users that have permissions for the given entity. Args: name: The name of the entity. entity_kind: The type (data_types.PermissionEntityKind) of the entity. auto_cc: The Auto CC type (data_types.AutoCCType) to filter on, or None. Returns: A list of user emails that have permission to access the given entity. """ if not name: return [] # Easy case: direct matches. direct_match_permissions = data_types.ExternalUserPermission.query( data_types.ExternalUserPermission.entity_kind == entity_kind, data_types.ExternalUserPermission.entity_name == name, ndb_utils.is_false(data_types.ExternalUserPermission.is_prefix), projection=[data_types.ExternalUserPermission.email]) if auto_cc is not None: direct_match_permissions = direct_match_permissions.filter( data_types.ExternalUserPermission.auto_cc == auto_cc) allowed_users = [ permission.email for permission in direct_match_permissions ] # Find all permissions where the prefix matches the fuzzer_name. # Unfortunately, Datastore doesn't give us an easy way of doing so. To iterate # through a smaller set than every single permission, get all permissions that # contain a prefix string <= than the actual fuzzer name and >= the first # character. prefix_match_permissions = data_types.ExternalUserPermission.query( data_types.ExternalUserPermission.entity_kind == entity_kind, data_types.ExternalUserPermission.entity_name <= name, data_types.ExternalUserPermission.entity_name >= name[0], ndb_utils.is_true(data_types.ExternalUserPermission.is_prefix), projection=[ data_types.ExternalUserPermission.email, data_types.ExternalUserPermission.entity_name ]) if auto_cc is not None: prefix_match_permissions = prefix_match_permissions.filter( data_types.ExternalUserPermission.auto_cc == auto_cc) for permission in prefix_match_permissions: if not permission.entity_name: # No external user should have an empty prefix (access to all # fuzzers/jobs). continue if name.startswith(permission.entity_name): allowed_users.append(permission.email) return sorted(allowed_users)
def query_testcase(project_name, crash_type, crash_state, security_flag, is_open): """Start a query for an associated testcase.""" return data_types.Testcase.query( data_types.Testcase.project_name == project_name, data_types.Testcase.crash_type == crash_type, data_types.Testcase.crash_state == crash_state, data_types.Testcase.security_flag == security_flag, data_types.Testcase.open == is_open, ndb_utils.is_false(data_types.Testcase.is_a_duplicate_flag)).order( -data_types.Testcase.timestamp).iter(limit=1, projection=[ 'bug_information', 'group_bug_information', ])
def get(self): """Handle a GET request.""" assert self.task # Create new tasks for the open reproducible test cases. for status in ['Processed', 'Duplicate']: testcases = data_types.Testcase.query( ndb_utils.is_true(data_types.Testcase.open), ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag), data_types.Testcase.status == status) for testcase in testcases: try: tasks.add_task(self.task, testcase.key.id(), testcase.job_type, queue=tasks.queue_for_testcase(testcase)) except Exception: logs.log_error('Failed to add task.') continue
def execute(args): """Query Testcases of the given projects, and conditionally file them to the corresponding GitHub repo.""" if not args.script_args: print('Need at least one project name (with -p) ' 'when running the backfiler.') return for project_name in args.script_args: print(f'Back filing project {project_name}') for testcase in data_types.Testcase.query( ndb_utils.is_true(data_types.Testcase.open), ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag), data_types.Testcase.status == 'Processed', data_types.Testcase.project_name == project_name, ): if not testcase.bug_information: print(f'Skip testcase without bugs: {testcase.key.id()}') continue print(f'Back filing testcase id: {testcase.key.id()}') if args.non_dry_run: oss_fuzz_github.file_issue(testcase)
def _is_crash_important(testcase): """Indicate if the crash is important to file.""" if not testcase.one_time_crasher_flag: # A reproducible crash is an important crash. return True if testcase.status != 'Processed': # A duplicate or unreproducible crash is not an important crash. return False # Testcase is unreproducible. Only those crashes that are crashing frequently # are important. if testcase.crash_type in UNREPRODUCIBLE_CRASH_IGNORE_CRASH_TYPES: return False # Ensure that there is no reproducible testcase in our group. if testcase.group_id: other_reproducible_testcase = data_types.Testcase.query( data_types.Testcase.group_id == testcase.group_id, ndb_utils.is_false(data_types.Testcase.one_time_crasher_flag)).get() if other_reproducible_testcase: # There is another reproducible testcase in our group. So, this crash is # not important. return False # Get crash statistics data on this unreproducible crash for last X days. last_hour = crash_stats.get_last_successful_hour() if not last_hour: # No crash stats available, skip. return False _, rows = crash_stats.get( end=last_hour, block='day', days=data_types.FILE_CONSISTENT_UNREPRODUCIBLE_TESTCASE_DEADLINE, group_by='reproducible_flag', where_clause=( 'crash_type = %s AND crash_state = %s AND security_flag = %s' % (json.dumps(testcase.crash_type), json.dumps(testcase.crash_state), json.dumps(testcase.security_flag))), group_having_clause='', sort_by='total_count', offset=0, limit=1) # Calculate total crash count and crash days count. crash_days_indices = set([]) total_crash_count = 0 for row in rows: if 'groups' not in row: continue total_crash_count += row['totalCount'] for group in row['groups']: for index in group['indices']: crash_days_indices.add(index['hour']) crash_days_count = len(crash_days_indices) # Only those unreproducible testcases are important that happened atleast once # everyday for the last X days and total crash count exceeded our threshold # limit. return (crash_days_count == data_types.FILE_CONSISTENT_UNREPRODUCIBLE_TESTCASE_DEADLINE and total_crash_count >= data_types.FILE_UNREPRODUCIBLE_TESTCASE_MIN_CRASH_THRESHOLD)