def get_all_from_query(query, **kwargs): """Return all entities based on the query by paging, to avoid query expirations on App Engine.""" if isinstance(query, ndb_patcher.Query): # Not necessary with ndb_patcher.Query. for result in query.iter(**kwargs): yield result return batch_size = kwargs.pop('batch_size', DEFAULT_BATCH_SIZE) kwargs['batch_size'] = batch_size while True: entities, cursor, more = query.fetch_page(batch_size, **kwargs) if not entities: break for entity in entities: yield entity kwargs['start_cursor'] = cursor if not more: # No more results to process, bail out. break # Free up some memory in between batches. del entities utils.python_gc()
def cleanup_task_state(): """Cleans state before and after a task is executed.""" # Cleanup stale processes. process_handler.cleanup_stale_processes() # Clear build urls, temp and testcase directories. shell.clear_build_urls_directory() shell.clear_crash_stacktraces_directory() shell.clear_testcase_directories() shell.clear_temp_directory() shell.clear_system_temp_directory() shell.clear_device_temp_directories() # Reset memory tool environment variables. environment.reset_current_memory_tool_options() # Call python's garbage collector. utils.python_gc()
def get(self): """Handle a get request.""" try: grouper.group_testcases() except: logs.log_error('Error occurred while grouping test cases.') return # Free up memory after group task run. utils.python_gc() # Get a list of jobs excluded from bug filing. excluded_jobs = _get_excluded_jobs() # Get a list of all jobs. This is used to filter testcases whose jobs have # been removed. all_jobs = data_handler.get_all_job_type_names() for testcase_id in data_handler.get_open_testcase_id_iterator(): try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue # Skip if testcase's job is removed. if testcase.job_type not in all_jobs: continue # Skip if testcase's job is in exclusions list. if testcase.job_type in excluded_jobs: continue # Skip if we are running progression task at this time. if testcase.get_metadata('progression_pending'): continue # If the testcase has a bug filed already, no triage is needed. if _is_bug_filed(testcase): continue # Check if the crash is important, i.e. it is either a reproducible crash # or an unreproducible crash happening frequently. if not _is_crash_important(testcase): continue # Require that all tasks like minimizaton, regression testing, etc have # finished. if not data_handler.critical_tasks_completed(testcase): continue # For testcases that are not part of a group, wait an additional time till # group task completes. # FIXME: In future, grouping might be dependent on regression range, so we # would have to add an additional wait time. if not testcase.group_id and not dates.time_has_expired( testcase.timestamp, hours=data_types.MIN_ELAPSED_TIME_SINCE_REPORT): continue # If this project does not have an associated issue tracker, we cannot # file this crash anywhere. issue_tracker = issue_tracker_utils.get_issue_tracker_for_testcase( testcase) if not issue_tracker: continue # If there are similar issues to this test case already filed or recently # closed, skip filing a duplicate bug. if _check_and_update_similar_bug(testcase, issue_tracker): continue # Clean up old triage messages that would be not applicable now. testcase.delete_metadata(TRIAGE_MESSAGE_KEY, update_testcase=False) # File the bug first and then create filed bug metadata. try: issue_filer.file_issue(testcase, issue_tracker) except Exception: logs.log_error('Failed to file issue for testcase %d.' % testcase_id) continue _create_filed_bug_metadata(testcase) logs.log('Filed new issue %s for testcase %d.' % (testcase.bug_information, testcase_id))
def _unpack_build(base_build_dir, build_dir, build_url, target_weights=None): """Unpacks a build from a build url into the build directory.""" # Track time taken to unpack builds so that it doesn't silently regress. start_time = time.time() # Free up memory. utils.python_gc() # Remove the current build. logs.log('Removing build directory %s.' % build_dir) if not shell.remove_directory(build_dir, recreate=True): logs.log_error('Unable to clear build directory %s.' % build_dir) _handle_unrecoverable_error_on_windows() return False # Decide whether to use cache build archives or not. use_cache = environment.get_value('CACHE_STORE', False) # Download build archive locally. build_local_archive = os.path.join(build_dir, os.path.basename(build_url)) # Make the disk space necessary for the archive available. archive_size = storage.get_download_file_size( build_url, build_local_archive, use_cache=True) if archive_size is not None and not _make_space(archive_size, base_build_dir): shell.clear_data_directories() logs.log_fatal_and_exit( 'Failed to make space for download. ' 'Cleared all data directories to free up space, exiting.') logs.log('Downloading build from url %s.' % build_url) try: storage.copy_file_from(build_url, build_local_archive, use_cache=use_cache) except: logs.log_error('Unable to download build url %s.' % build_url) return False unpack_everything = environment.get_value('UNPACK_ALL_FUZZ_TARGETS_AND_FILES') if not unpack_everything: # For fuzzing, pick a random fuzz target so that we only un-archive that # particular fuzz target and its dependencies and save disk space. # If we are going to unpack everythng in archive based on # |UNPACK_ALL_FUZZ_TARGETS_AND_FILES| in the job defition, then don't set a # random fuzz target before we've unpacked the build. It won't actually save # us anything in this case and can be really expensive for large builds # (such as Chrome OS). Defer setting it until after the build has been # unpacked. _set_random_fuzz_target_for_fuzzing_if_needed( _get_fuzz_targets_from_archive(build_local_archive), target_weights) # Actual list of files to unpack can be smaller if we are only unarchiving # a particular fuzz target. file_match_callback = _get_file_match_callback() assert not (unpack_everything and file_match_callback is not None) if not _make_space_for_build(build_local_archive, base_build_dir, file_match_callback): shell.clear_data_directories() logs.log_fatal_and_exit( 'Failed to make space for build. ' 'Cleared all data directories to free up space, exiting.') # Unpack the local build archive. logs.log('Unpacking build archive %s.' % build_local_archive) trusted = not utils.is_oss_fuzz() try: archive.unpack( build_local_archive, build_dir, trusted=trusted, file_match_callback=file_match_callback) except: logs.log_error('Unable to unpack build archive %s.' % build_local_archive) return False if unpack_everything: # Set a random fuzz target now that the build has been unpacked, if we # didn't set one earlier. _set_random_fuzz_target_for_fuzzing_if_needed( _get_fuzz_targets_from_dir(build_dir), target_weights) # If this is partial build due to selected build files, then mark it as such # so that it is not re-used. if file_match_callback: partial_build_file_path = os.path.join(build_dir, PARTIAL_BUILD_FILE) utils.write_data_to_file('', partial_build_file_path) # No point in keeping the archive around. shell.remove_file(build_local_archive) end_time = time.time() elapsed_time = end_time - start_time log_func = logs.log_warn if elapsed_time > UNPACK_TIME_LIMIT else logs.log log_func('Build took %0.02f minutes to unpack.' % (elapsed_time / 60.)) return True
def cleanup_testcases_and_issues(): """Clean up unneeded open testcases and their associated issues.""" jobs = data_handler.get_all_job_type_names() testcase_keys = ndb_utils.get_all_from_query( data_types.Testcase.query( ndb_utils.is_false(data_types.Testcase.triaged)), keys_only=True) top_crashes_by_project_and_platform_map = ( get_top_crashes_for_all_projects_and_platforms()) utils.python_gc() testcases_processed = 0 empty_issue_tracker_policy = issue_tracker_policy.get_empty() for testcase_key in testcase_keys: testcase_id = testcase_key.id() try: testcase = data_handler.get_testcase_by_id(testcase_id) except errors.InvalidTestcaseError: # Already deleted. continue logs.log('Processing testcase %d.' % testcase_id) issue = issue_tracker_utils.get_issue_for_testcase(testcase) policy = issue_tracker_utils.get_issue_tracker_policy_for_testcase(testcase) if not policy: policy = empty_issue_tracker_policy # Issue updates. update_os_labels(policy, testcase, issue) update_fuzz_blocker_label(policy, testcase, issue, top_crashes_by_project_and_platform_map) update_component_labels(testcase, issue) update_issue_ccs_from_owners_file(policy, testcase, issue) update_issue_owner_and_ccs_from_predator_results(policy, testcase, issue) update_issue_labels_for_flaky_testcase(policy, testcase, issue) # Testcase marking rules. mark_duplicate_testcase_as_closed_with_no_issue(testcase) mark_issue_as_closed_if_testcase_is_fixed(policy, testcase, issue) mark_testcase_as_closed_if_issue_is_closed(policy, testcase, issue) mark_testcase_as_closed_if_job_is_invalid(testcase, jobs) mark_unreproducible_testcase_as_fixed_if_issue_is_closed(testcase, issue) mark_unreproducible_testcase_and_issue_as_closed_after_deadline( policy, testcase, issue) # Notification, to be done at end after testcase state is updated from # previous rules. notify_closed_issue_if_testcase_is_open(policy, testcase, issue) notify_issue_if_testcase_is_invalid(policy, testcase, issue) notify_uploader_when_testcase_is_processed(policy, testcase, issue) # Mark testcase as triage complete if both testcase and associated issue # are closed. This also need to be done before the deletion rules. mark_testcase_as_triaged_if_needed(testcase, issue) # Testcase deletion rules. delete_unreproducible_testcase_with_no_issue(testcase) testcases_processed += 1 if testcases_processed % 100 == 0: utils.python_gc()