def download_repo_prop_if_needed(symbols_directory, build_id, cache_target, targets_with_type_and_san, cache_type): """Downloads the repo.prop for a branch""" artifact_file_name = 'repo.prop' symbols_archive_filename = get_repo_prop_archive_filename( build_id, cache_target) output_filename_override = symbols_archive_filename # We create our own build_params for cache build_params = { 'build_id': build_id, 'target': cache_target, 'type': cache_type } build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) download_artifact_if_needed(build_id, symbols_directory, symbols_archive_path, targets_with_type_and_san, artifact_file_name, output_filename_override, build_params, build_params_check_path) if not os.path.exists(symbols_archive_path): logs.log_error('Unable to locate repo.prop %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)
def download_system_symbols_if_needed(symbols_directory): """Download system libraries from |SYMBOLS_URL| and cache locally.""" if not should_download_symbols(): return # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') build_id = build_params.get('build_id') target = build_params.get('target') build_type = build_params.get('type') if not build_id or not target or not build_type: logs.log_error('Null build parameters found, exiting.') return symbols_archive_filename = f'{target}-symbols-{build_id}.zip' artifact_file_name = symbols_archive_filename output_filename_override = None # Include type and sanitizer information in the target. tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') target_with_type_and_san = f'{target}-{build_type}' if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += f'_{tool_suffix}' targets_with_type_and_san = [target_with_type_and_san] symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) download_artifact_if_needed(build_id, symbols_directory, symbols_archive_path, targets_with_type_and_san, artifact_file_name, output_filename_override, build_params, build_params_check_path) if not os.path.exists(symbols_archive_path): logs.log_error('Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)
def download_system_symbols_if_needed(symbols_directory): """Download system libraries from |SYMBOLS_URL| and cache locally.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. if environment.get_value('LOCAL_DEVELOPMENT'): return # When running reproduce tool locally, we do not have access to the cloud # storage bucket with the symbols. In this case, just bail out. if environment.get_value('REPRODUCE_TOOL'): return # We have archived symbols for google builds only. if not settings.is_google_device(): return # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_id = build_params.get('build_id') target = build_params.get('target') type = build_params.get('type') if not build_id or not target or not type: logs.log_error('Null build parameters found, exiting.') return # Check if we already have the symbols in cache. build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') cached_build_params = utils.read_data_from_file( build_params_check_path, eval_data=True) if cached_build_params and cmp(cached_build_params, build_params) == 0: # No work to do, same system symbols already in cache. return symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id) symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) # Delete existing symbols directory first. shell.remove_directory(symbols_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( symbols_archive_path, update_modification_time_on_access=False) if not found_in_cache: # Include type and sanitizer information in the target. target_with_type_and_san = '%s-%s' % (target, type) tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += '_%s' % tool_suffix # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, symbols_archive_filename, symbols_directory) if not os.path.exists(symbols_archive_path): logs.log_error( 'Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)
def download_system_symbols_if_needed(symbols_directory, is_kernel=False): """Download system libraries from |SYMBOLS_URL| and cache locally.""" # For local testing, we do not have access to the cloud storage bucket with # the symbols. In this case, just bail out. if environment.get_value('LOCAL_DEVELOPMENT'): return # When running reproduce tool locally, we do not have access to the cloud # storage bucket with the symbols. In this case, just bail out. if environment.get_value('REPRODUCE_TOOL'): return # We have archived symbols for google builds only. if not settings.is_google_device(): return # For Android kernel we want to get the repro.prop # Note: kasan and non-kasan kernel should have the same repo.prop for a given # build_id. if is_kernel: _, build_id = kernel_utils.get_kernel_hash_and_build_id() target = kernel_utils.get_kernel_name() if not build_id or not target: logs.log_error('Could not get kernel parameters, exiting.') return artifact_file_name = 'repo.prop' symbols_archive_filename = get_symbols_archive_filename(build_id, target) output_filename_override = symbols_archive_filename # We create our own build_params for cache build_params = {'build_id': build_id, 'target': target, 'type': 'kernel'} else: # Get the build fingerprint parameters. build_params = settings.get_build_parameters() if not build_params: logs.log_error('Unable to determine build parameters.') return build_id = build_params.get('build_id') target = build_params.get('target') build_type = build_params.get('type') if not build_id or not target or not build_type: logs.log_error('Null build parameters found, exiting.') return symbols_archive_filename = '%s-symbols-%s.zip' % (target, build_id) artifact_file_name = symbols_archive_filename output_filename_override = None # Check if we already have the symbols in cache. build_params_check_path = os.path.join(symbols_directory, '.cached_build_params') cached_build_params = utils.read_data_from_file( build_params_check_path, eval_data=True) if cached_build_params and cached_build_params == build_params: # No work to do, same system symbols already in cache. return symbols_archive_path = os.path.join(symbols_directory, symbols_archive_filename) # Delete existing symbols directory first. shell.remove_directory(symbols_directory, recreate=True) # Fetch symbol file from cloud storage cache (if available). found_in_cache = storage.get_file_from_cache_if_exists( symbols_archive_path, update_modification_time_on_access=False) if not found_in_cache: tool_suffix = environment.get_value('SANITIZER_TOOL_NAME') if is_kernel: # Some kernels are just 'kernel', some are kernel_target if tool_suffix: targets_with_type_and_san = [ 'kernel_%s' % tool_suffix, 'kernel_%s_%s' % (tool_suffix, target) ] else: targets_with_type_and_san = ['kernel', 'kernel_%s' % target] else: # Include type and sanitizer information in the target. target_with_type_and_san = '%s-%s' % (target, build_type) if tool_suffix and not tool_suffix in target_with_type_and_san: target_with_type_and_san += '_%s' % tool_suffix targets_with_type_and_san = [target_with_type_and_san] for target_with_type_and_san in targets_with_type_and_san: # Fetch the artifact now. fetch_artifact.get(build_id, target_with_type_and_san, artifact_file_name, symbols_directory, output_filename_override) if os.path.exists(symbols_archive_path): break if not os.path.exists(symbols_archive_path): logs.log_error( 'Unable to locate symbols archive %s.' % symbols_archive_path) return # Store the artifact for later use or for use by other bots. storage.store_file_in_cache(symbols_archive_path) # repo.prop is not a zip archive. if not is_kernel: archive.unpack(symbols_archive_path, symbols_directory, trusted=True) shell.remove_file(symbols_archive_path) utils.write_data_to_file(build_params, build_params_check_path)