def get(self): """Handle a get request.""" fuzzer_logs_bucket = fuzzer_logs.get_bucket() fuzzers = list(data_types.Fuzzer.query().order(data_types.Fuzzer.name)) jobs = data_handler.get_all_job_type_names() corpora = [ bundle.name for bundle in data_types.DataBundle.query().order( data_types.DataBundle.name) ] privileged = access.has_access(need_privileged_access=True) # Unprivileged users can't download fuzzers, so hide the download keys. if not privileged: for fuzzer in fuzzers: fuzzer.blobstore_key = '' template_values = { 'privileged': privileged, 'fuzzers': fuzzers, 'fuzzerLogsBucket': fuzzer_logs_bucket, 'fieldValues': { 'corpora': corpora, 'jobs': jobs, 'uploadInfo': gcs.prepare_blob_upload()._asdict(), 'csrfToken': form.generate_csrf_token(), } } self.render('fuzzers.html', template_values)
def _allowed_entities_for_user(user_email, entity_kind): """Return the entity names that the given user can access. Args: user_email: The email of the user. entity_kind: The type (data_types.PermissionEntityKind) of the entity. Returns: A list of entity names that the user has access to. """ if not user_email: return [] allowed = [] permissions = _get_permissions_query_for_user(user_email, entity_kind) if entity_kind == data_types.PermissionEntityKind.FUZZER: all_names = data_handler.get_all_fuzzer_names_including_children() else: all_names = data_handler.get_all_job_type_names() for permission in permissions: if permission.is_prefix: allowed.extend(_expand_prefix(all_names, permission.entity_name)) elif permission.entity_name in all_names: allowed.append(permission.entity_name) return sorted(allowed)
def get(self): """Handle a GET request.""" # Create a list of externally contributed fuzzers. user_email = helpers.get_user_email() if access.has_access(): # User is an internal user of ClusterFuzz (eg: ClusterFuzz developer). fuzzers_list = ( data_handler.get_all_fuzzer_names_including_children( include_parents=True)) jobs_list = data_handler.get_all_job_type_names() else: # User is an external user of ClusterFuzz (eg: non-Chrome dev who # submitted a fuzzer or someone with a project in OSS-Fuzz). fuzzers_list = external_users.allowed_fuzzers_for_user( user_email, include_from_jobs=True, include_parents=True) if not fuzzers_list: # User doesn't actually have access to any fuzzers. raise helpers.AccessDeniedException() jobs_list = external_users.allowed_jobs_for_user(user_email) fuzzers_list.sort() jobs_list.sort() result = { 'info': { 'fuzzers': fuzzers_list, 'jobs': jobs_list, } } self.render('fuzzer-stats.html', result)
def get(self): """Handle a GET request.""" # pylint: disable=unexpected-keyword-arg # Memoize all project and job names. _ = data_handler.get_all_project_names(__memoize_force__=True) _ = data_handler.get_all_job_type_names(__memoize_force__=True) # Memoize both variants of get_all_fuzzer_names_including_children. _ = data_handler.get_all_fuzzer_names_including_children( include_parents=True, __memoize_force__=True) _ = data_handler.get_all_fuzzer_names_including_children( __memoize_force__=True) # Memoize expensive testcase attribute calls. for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue blobs.get_blob_size(testcase.fuzzed_keys) blobs.get_blob_size(testcase.minimized_keys) self.response.headers['Content-Type'] = 'text/plain' self.response.out.write('OK') self.response.set_status(200)
def cleanup_testcases_and_issues(): """Clean up unneeded open testcases and their associated issues.""" jobs = data_handler.get_all_job_type_names() testcase_keys = ndb_utils.get_all_from_query(data_types.Testcase.query( ndb_utils.is_false(data_types.Testcase.triaged)), keys_only=True) top_crashes_by_project_and_platform_map = ( get_top_crashes_for_all_projects_and_platforms()) for testcase_key in testcase_keys: try: testcase = data_handler.get_testcase_by_id(testcase_key.id()) except errors.InvalidTestcaseError: # Already deleted. continue issue = issue_tracker_utils.get_issue_for_testcase(testcase) policy = issue_tracker_utils.get_issue_tracker_policy_for_testcase( testcase) if not policy: policy = issue_tracker_policy.get_empty() # Issue updates. update_os_labels(policy, testcase, issue) update_fuzz_blocker_label(policy, testcase, issue, top_crashes_by_project_and_platform_map) update_component_labels(testcase, issue) update_issue_ccs_from_owners_file(policy, testcase, issue) update_issue_owner_and_ccs_from_predator_results( policy, testcase, issue) update_issue_labels_for_flaky_testcase(policy, testcase, issue) # Testcase marking rules. mark_duplicate_testcase_as_closed_with_no_issue(testcase) mark_issue_as_closed_if_testcase_is_fixed(policy, testcase, issue) mark_testcase_as_closed_if_issue_is_closed(policy, testcase, issue) mark_testcase_as_closed_if_job_is_invalid(testcase, jobs) mark_unreproducible_testcase_as_fixed_if_issue_is_closed( testcase, issue) mark_unreproducible_testcase_and_issue_as_closed_after_deadline( policy, testcase, issue) # Notification, to be done at end after testcase state is updated from # previous rules. notify_closed_issue_if_testcase_is_open(policy, testcase, issue) notify_issue_if_testcase_is_invalid(policy, testcase, issue) notify_uploader_when_testcase_is_processed(policy, testcase, issue) # Mark testcase as triage complete if both testcase and associated issue # are closed. This also need to be done before the deletion rules. mark_testcase_as_triaged_if_needed(testcase, issue) # Testcase deletion rules. delete_unreproducible_testcase_with_no_issue(testcase)
def get(self): """Handles get request.""" email = helpers.get_user_email() if not email: raise helpers.AccessDeniedException() is_privileged_or_domain_user = access.has_access( need_privileged_access=False) if is_privileged_or_domain_user or _is_uploader_allowed(email): # Privileged, domain and upload users can see all job and fuzzer names. allowed_jobs = data_handler.get_all_job_type_names() allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children( include_parents=True) else: # Check if this is an external user with access to certain fuzzers/jobs. allowed_jobs = external_users.allowed_jobs_for_user(email) allowed_fuzzers = external_users.allowed_fuzzers_for_user( email, include_from_jobs=True) if not allowed_fuzzers and not allowed_jobs: raise helpers.AccessDeniedException() has_issue_tracker = bool(data_handler.get_issue_tracker_name()) result, params = get_result() return self.render( 'upload.html', { 'fieldValues': { 'blackboxFuzzers': filter_blackbox_fuzzers(allowed_fuzzers), 'jobs': allowed_jobs, 'libfuzzerTargets': filter_target_names(allowed_fuzzers, 'libFuzzer'), 'aflTargets': filter_target_names(allowed_fuzzers, 'afl'), 'honggfuzzTargets': filter_target_names(allowed_fuzzers, 'honggfuzz'), 'isChromium': utils.is_chromium(), 'sandboxedJobs': data_types.INTERNAL_SANDBOXED_JOB_TYPES, 'csrfToken': form.generate_csrf_token(), 'isExternalUser': not is_privileged_or_domain_user, 'uploadInfo': gcs.prepare_blob_upload()._asdict(), 'hasIssueTracker': has_issue_tracker, }, 'params': params, 'result': result })
def get(self): """Handles get request.""" email = helpers.get_user_email() if not email: raise helpers.AccessDeniedException() is_privileged_or_domain_user = access.has_access( need_privileged_access=False) if is_privileged_or_domain_user or _is_uploader_allowed(email): # Privileged, domain and upload users can see all job and fuzzer names. allowed_jobs = data_handler.get_all_job_type_names() allowed_fuzzers = data_handler.get_all_fuzzer_names_including_children( include_parents=True) else: # Check if this is an external user with access to certain fuzzers/jobs. allowed_jobs = external_users.allowed_jobs_for_user(email) allowed_fuzzers = external_users.allowed_fuzzers_for_user( email, include_from_jobs=True) if not allowed_fuzzers and not allowed_jobs: raise helpers.AccessDeniedException() has_issue_tracker = bool(data_handler.get_issue_tracker_name()) result, params = get_result(self) self.render( "upload.html", { "fieldValues": { "jobs": allowed_jobs, "libfuzzerTargets": filter_target_names(allowed_fuzzers, "libFuzzer"), "aflTargets": filter_target_names(allowed_fuzzers, "afl"), "isChromium": utils.is_chromium(), "sandboxedJobs": data_types.INTERNAL_SANDBOXED_JOB_TYPES, "csrfToken": form.generate_csrf_token(), "isExternalUser": not is_privileged_or_domain_user, "uploadInfo": gcs.prepare_blob_upload()._asdict(), "hasIssueTracker": has_issue_tracker, }, "params": params, "result": result, }, )
def get(self): """Handle a GET request.""" # pylint: disable=unexpected-keyword-arg _ = data_handler.get_all_project_names(__memoize_force__=True) _ = data_handler.get_all_job_type_names(__memoize_force__=True) # Memorize both variants of get_all_fuzzer_names_including_children. _ = data_handler.get_all_fuzzer_names_including_children( include_parents=True, __memoize_force__=True) _ = data_handler.get_all_fuzzer_names_including_children( __memoize_force__=True) self.response.headers['Content-Type'] = 'text/plain' self.response.out.write('OK') self.response.set_status(200)
def get(self): """Get and render the testcase list in HTML.""" result, params = get_result(self) field_values = { 'projects': data_handler.get_all_project_names(), 'fuzzers': data_handler.get_all_fuzzer_names_including_children( include_parents=True), 'jobs': data_handler.get_all_job_type_names(), 'shouldShowImpact': utils.is_chromium() } self.render('testcase-list.html', { 'fieldValues': field_values, 'result': result, 'params': params })
def get(self): """Handle a GET request.""" project = request.get('project') if access.has_access(): # User is an internal user of ClusterFuzz (eg: ClusterFuzz developer). # Show all projects in the list, since this allows user to pick another # project as needed. projects_list = data_handler.get_all_project_names() # Filter fuzzers and job list if a project is provided. fuzzers_list = ( data_handler.get_all_fuzzer_names_including_children( include_parents=True, project=project)) jobs_list = data_handler.get_all_job_type_names(project=project) else: # User is an external user of ClusterFuzz (eg: non-Chrome dev who # submitted a fuzzer or someone with a project in OSS-Fuzz). user_email = helpers.get_user_email() # TODO(aarya): Filter fuzzer and job if |project| is provided. fuzzers_list = sorted( external_users.allowed_fuzzers_for_user(user_email, include_from_jobs=True, include_parents=True)) if not fuzzers_list: # User doesn't actually have access to any fuzzers. raise helpers.AccessDeniedException( "You don't have access to any fuzzers.") jobs_list = sorted( external_users.allowed_jobs_for_user(user_email)) projects_list = sorted( {data_handler.get_project_name(job) for job in jobs_list}) result = { 'projects': projects_list, 'fuzzers': fuzzers_list, 'jobs': jobs_list, } return self.render_json(result)
def get(self): """Get and render the crash stats in HTML.""" result, params = get_result(self) field_values = { 'fuzzers': data_handler.get_all_fuzzer_names_including_children( include_parents=True), 'jobs': data_handler.get_all_job_type_names(), 'platforms': get_all_platforms(), 'projects': data_handler.get_all_project_names(), 'minHour': crash_stats_common.get_min_hour(), 'maxHour': crash_stats_common.get_max_hour() } self.render('crash-stats.html', { 'result': result, 'fieldValues': field_values, 'params': params })
def get(self): """Handle a get request.""" try: grouper.group_testcases() except: logs.log_error('Error occurred while grouping test cases.') return # Free up memory after group task run. utils.python_gc() # Get a list of jobs excluded from bug filing. excluded_jobs = _get_excluded_jobs() # Get a list of all jobs. This is used to filter testcases whose jobs have # been removed. all_jobs = data_handler.get_all_job_type_names() for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue # Skip if testcase's job is removed. if testcase.job_type not in all_jobs: continue # Skip if testcase's job is in exclusions list. if testcase.job_type in excluded_jobs: continue # Skip if we are running progression task at this time. if testcase.get_metadata('progression_pending'): continue # If the testcase has a bug filed already, no triage is needed. if _is_bug_filed(testcase): continue # Check if the crash is important, i.e. it is either a reproducible crash # or an unreproducible crash happening frequently. if not _is_crash_important(testcase): continue # Require that all tasks like minimizaton, regression testing, etc have # finished. if not data_handler.critical_tasks_completed(testcase): continue # For testcases that are not part of a group, wait an additional time till # group task completes. # FIXME: In future, grouping might be dependent on regression range, so we # would have to add an additional wait time. if not testcase.group_id and not dates.time_has_expired( testcase.timestamp, hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT): continue # If this project does not have an associated issue tracker, we cannot # file this crash anywhere. issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( testcase) if not issue_tracker: continue # If there are similar issues to this test case already filed or recently # closed, skip filing a duplicate bug. if _check_and_update_similar_bug(testcase, issue_tracker): continue # Clean up old triage messages that would be not applicable now. testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False) # File the bug first and then create filed bug metadata. try: issue_filer.file_issue(testcase, issue_tracker) except Exception: logs.log_error('Failed to file issue for testcase %d.' % testcase_id) continue _create_filed_bug_metadata(testcase) logs.log('Filed new issue %s for testcase %d.' % (testcase.bug_information, testcase_id))
def post(self): """Handle a post request.""" name = self.request.get('name') if not name: raise helpers.EarlyExitException('Please give this job a name!', 400) if not data_types.Job.VALID_NAME_REGEX.match(name): raise helpers.EarlyExitException( 'Job name can only contain letters, numbers, dashes and underscores.', 400) fuzzers = self.request.get('fuzzers', []).split(',') templates = self.request.get('templates', '').splitlines() for template in templates: if not data_types.JobTemplate.query( data_types.JobTemplate.name == template).get(): raise helpers.EarlyExitException( 'Invalid template name(s) specified.', 400) new_platform = self.request.get('platform') if not new_platform or new_platform == 'undefined': raise helpers.EarlyExitException('No platform provided for job.', 400) description = self.request.get('description', '') environment_string = self.request.get('environment_string', '') previous_custom_binary_revision = 0 job = data_types.Job.query(data_types.Job.name == name).get() recreate_fuzzer_mappings = False if not job: job = data_types.Job() else: previous_custom_binary_revision = job.custom_binary_revision if previous_custom_binary_revision is None: previous_custom_binary_revision = 0 if new_platform != job.platform: # The rare case of modifying a job's platform causes many problems with # task selection. If a job is leased from the old queue, the task will # be recreated in the correct queue at lease time. Fuzzer mappings must # be purged and recreated, since they depend on the job's platform. recreate_fuzzer_mappings = True job.name = name job.platform = new_platform job.description = description job.environment_string = environment_string job.templates = templates blob_info = self.get_upload() if blob_info: job.custom_binary_key = str(blob_info.key()) job.custom_binary_filename = blob_info.filename job.custom_binary_revision = previous_custom_binary_revision + 1 if job.custom_binary_key and 'CUSTOM_BINARY' not in job.environment_string: job.environment_string += '\nCUSTOM_BINARY = True' job.put() fuzzer_selection.update_mappings_for_job(job, fuzzers) if recreate_fuzzer_mappings: fuzzer_selection.update_platform_for_job(name, new_platform) # pylint: disable=unexpected-keyword-arg _ = data_handler.get_all_job_type_names(__memoize_force__=True) helpers.log('Job created %s' % name, helpers.MODIFY_OPERATION) template_values = { 'title': 'Success', 'message': ('Job %s is successfully updated. ' 'Redirecting back to jobs page...') % name, 'redirect_url': '/jobs', } self.render('message.html', template_values)
def do_post(self): """Upload a testcase.""" email = helpers.get_user_email() testcase_id = request.get('testcaseId') uploaded_file = self.get_upload() if testcase_id and not uploaded_file: testcase = helpers.get_testcase(testcase_id) if not access.can_user_access_testcase(testcase): raise helpers.AccessDeniedException() # Use minimized testcase for upload (if available). key = (testcase.minimized_keys if testcase.minimized_keys and testcase.minimized_keys != 'NA' else testcase.fuzzed_keys) uploaded_file = blobs.get_blob_info(key) # Extract filename part from blob. uploaded_file.filename = os.path.basename( uploaded_file.filename.replace('\\', os.sep)) job_type = request.get('job') if not job_type: raise helpers.EarlyExitException('Missing job name.', 400) if (not data_types.Job.VALID_NAME_REGEX.match(job_type) or not job_type in data_handler.get_all_job_type_names()): raise helpers.EarlyExitException('Invalid job name.', 400) fuzzer_name = request.get('fuzzer') job_type_lowercase = job_type.lower() if 'libfuzzer' in job_type_lowercase: fuzzer_name = 'libFuzzer' elif 'afl' in job_type_lowercase: fuzzer_name = 'afl' elif 'honggfuzz' in job_type_lowercase: fuzzer_name = 'honggfuzz' is_engine_job = fuzzer_name and environment.is_engine_fuzzer_job( job_type) target_name = request.get('target') if not is_engine_job and target_name: raise helpers.EarlyExitException( 'Target name is not applicable to non-engine jobs (AFL, libFuzzer).', 400) if is_engine_job and not target_name: raise helpers.EarlyExitException( 'Missing target name for engine job (AFL, libFuzzer).', 400) if (target_name and not data_types.Fuzzer.VALID_NAME_REGEX.match(target_name)): raise helpers.EarlyExitException('Invalid target name.', 400) fully_qualified_fuzzer_name = '' if is_engine_job and target_name: fully_qualified_fuzzer_name, target_name = find_fuzz_target( fuzzer_name, target_name, job_type) if (not access.has_access(need_privileged_access=False, job_type=job_type, fuzzer_name=(fully_qualified_fuzzer_name or fuzzer_name)) and not _is_uploader_allowed(email)): raise helpers.AccessDeniedException() multiple_testcases = bool(request.get('multiple')) http_flag = bool(request.get('http')) high_end_job = bool(request.get('highEnd')) bug_information = request.get('issue') crash_revision = request.get('revision') timeout = request.get('timeout') retries = request.get('retries') bug_summary_update_flag = bool(request.get('updateIssue')) quiet_flag = bool(request.get('quiet')) additional_arguments = request.get('args') app_launch_command = request.get('cmd') platform_id = request.get('platform') issue_labels = request.get('issue_labels') gestures = request.get('gestures') or '[]' testcase_metadata = request.get('metadata', {}) if testcase_metadata: try: testcase_metadata = json.loads(testcase_metadata) except Exception: raise helpers.EarlyExitException('Invalid metadata JSON.', 400) if not isinstance(testcase_metadata, dict): raise helpers.EarlyExitException( 'Metadata is not a JSON object.', 400) if issue_labels: testcase_metadata['issue_labels'] = issue_labels try: gestures = ast.literal_eval(gestures) except Exception: raise helpers.EarlyExitException('Failed to parse gestures.', 400) archive_state = 0 bundled = False file_path_input = '' # Certain modifications such as app launch command, issue updates are only # allowed for privileged users. privileged_user = access.has_access(need_privileged_access=True) if not privileged_user: if bug_information or bug_summary_update_flag: raise helpers.EarlyExitException( 'You are not privileged to update existing issues.', 400) need_privileged_access = utils.string_is_true( data_handler.get_value_from_job_definition( job_type, 'PRIVILEGED_ACCESS')) if need_privileged_access: raise helpers.EarlyExitException( 'You are not privileged to run this job type.', 400) if app_launch_command: raise helpers.EarlyExitException( 'You are not privileged to run arbitrary launch commands.', 400) if (testcase_metadata and not _allow_unprivileged_metadata(testcase_metadata)): raise helpers.EarlyExitException( 'You are not privileged to set testcase metadata.', 400) if additional_arguments: raise helpers.EarlyExitException( 'You are not privileged to add command-line arguments.', 400) if gestures: raise helpers.EarlyExitException( 'You are not privileged to run arbitrary gestures.', 400) # TODO(aarya): Remove once AFL is migrated to engine pipeline. if target_name: additional_arguments = '%TESTCASE%' if crash_revision and crash_revision.isdigit(): crash_revision = int(crash_revision) else: crash_revision = 0 if bug_information == '0': # Auto-recover from this bad input. bug_information = None if bug_information and not bug_information.isdigit(): raise helpers.EarlyExitException('Bug is not a number.', 400) if not timeout: timeout = 0 elif not timeout.isdigit() or timeout == '0': raise helpers.EarlyExitException( 'Testcase timeout must be a number greater than 0.', 400) else: timeout = int(timeout) if timeout > 120: raise helpers.EarlyExitException( 'Testcase timeout may not be greater than 120 seconds.', 400) if retries: if retries.isdigit(): retries = int(retries) else: retries = None if retries is None or retries > MAX_RETRIES: raise helpers.EarlyExitException( 'Testcase retries must be a number less than %d.' % MAX_RETRIES, 400) else: retries = None job_queue = tasks.queue_for_job(job_type, is_high_end=high_end_job) if uploaded_file is not None: filename = ''.join([ x for x in uploaded_file.filename if x not in ' ;/?:@&=+$,{}|<>()\\' ]) key = str(uploaded_file.key()) if archive.is_archive(filename): archive_state = data_types.ArchiveStatus.FUZZED if archive_state: if multiple_testcases: # Create a job to unpack an archive. metadata = data_types.BundledArchiveMetadata() metadata.blobstore_key = key metadata.timeout = timeout metadata.job_queue = job_queue metadata.job_type = job_type metadata.http_flag = http_flag metadata.archive_filename = filename metadata.uploader_email = email metadata.gestures = gestures metadata.crash_revision = crash_revision metadata.additional_arguments = additional_arguments metadata.bug_information = bug_information metadata.platform_id = platform_id metadata.app_launch_command = app_launch_command metadata.fuzzer_name = fuzzer_name metadata.overridden_fuzzer_name = fully_qualified_fuzzer_name metadata.fuzzer_binary_name = target_name metadata.put() tasks.add_task('unpack', str(metadata.key.id()), job_type, queue=tasks.queue_for_job(job_type)) # Create a testcase metadata object to show the user their upload. upload_metadata = data_types.TestcaseUploadMetadata() upload_metadata.timestamp = datetime.datetime.utcnow() upload_metadata.filename = filename upload_metadata.blobstore_key = key upload_metadata.original_blobstore_key = key upload_metadata.status = 'Pending' upload_metadata.bundled = True upload_metadata.uploader_email = email upload_metadata.retries = retries upload_metadata.bug_summary_update_flag = bug_summary_update_flag upload_metadata.quiet_flag = quiet_flag upload_metadata.additional_metadata_string = json.dumps( testcase_metadata) upload_metadata.put() helpers.log('Uploaded multiple testcases.', helpers.VIEW_OPERATION) return self.render_json({'multiple': True}) file_path_input = guess_input_file(uploaded_file, filename) if not file_path_input: raise helpers.EarlyExitException(( "Unable to detect which file to launch. The main file\'s name " 'must contain either of %s.' % str(RUN_FILE_PATTERNS)), 400) else: raise helpers.EarlyExitException('Please select a file to upload.', 400) testcase_id = data_handler.create_user_uploaded_testcase( key, key, archive_state, filename, file_path_input, timeout, job_type, job_queue, http_flag, gestures, additional_arguments, bug_information, crash_revision, email, platform_id, app_launch_command, fuzzer_name, fully_qualified_fuzzer_name, target_name, bundled, retries, bug_summary_update_flag, quiet_flag, additional_metadata=testcase_metadata) if not quiet_flag: testcase = data_handler.get_testcase_by_id(testcase_id) issue = issue_tracker_utils.get_issue_for_testcase(testcase) if issue: report_url = data_handler.TESTCASE_REPORT_URL.format( domain=data_handler.get_domain(), testcase_id=testcase_id) comment = ('ClusterFuzz is analyzing your testcase. ' 'Developers can follow the progress at %s.' % report_url) issue.save(new_comment=comment) helpers.log('Uploaded testcase %s' % testcase_id, helpers.VIEW_OPERATION) return self.render_json({'id': '%s' % testcase_id})