def _zip_file(file_info):
"""Zip the file to reduce the file size.
@param file_info: A ResultInfo object containing summary for the file to be
shrunk.
"""
utils_lib.LOG('Compressing file %s' % file_info.path)
parent_result_info = file_info.parent_result_info
new_name = file_info.name + '.tgz'
new_path = os.path.join(os.path.dirname(file_info.path), new_name)
if os.path.exists(new_path):
utils_lib.LOG('File %s already exists, removing...' % new_path)
if not throttler_lib.try_delete_file_on_disk(new_path):
return
parent_result_info.remove_file(new_name)
with tarfile.open(new_path, 'w:gz') as tar:
tar.add(file_info.path, arcname=os.path.basename(file_info.path))
stat = os.stat(file_info.path)
if not throttler_lib.try_delete_file_on_disk(file_info.path):
# Clean up the intermediate file.
throttler_lib.try_delete_file_on_disk(new_path)
utils_lib.LOG('Failed to compress %s' % file_info.path)
return
# Modify the new file's timestamp to the old one.
os.utime(new_path, (stat.st_atime, stat.st_mtime))
# Get the original file size before compression.
original_size = file_info.original_size
parent_result_info.remove_file(file_info.name)
parent_result_info.add_file(new_name)
new_file_info = parent_result_info.get_file(new_name)
# Set the original size to be the size before compression.
new_file_info.original_size = original_size
# Set the trimmed size to be the physical file size of the compressed file.
new_file_info.trimmed_size = new_file_info.size
def _trim_file(file_info, file_size_limit_byte):
"""Remove the file content in the middle to reduce the file size.
@param file_info: A ResultInfo object containing summary for the file to be
shrunk.
@param file_size_limit_byte: Maximum file size in bytes after trimming.
"""
utils_lib.LOG('Trimming file %s to reduce size from %d bytes to %d bytes' %
(file_info.path, file_info.original_size,
file_size_limit_byte))
new_path = os.path.join(os.path.dirname(file_info.path),
file_info.name + '_trimmed')
original_size_bytes = file_info.original_size
with open(new_path, 'w') as new_file, open(file_info.path) as old_file:
# Read the beginning part of the old file, if it's already started with
# TRIMMED_FILE_HEADER, no need to add the header again.
header = old_file.read(len(TRIMMED_FILE_HEADER))
if header != TRIMMED_FILE_HEADER:
new_file.write(TRIMMED_FILE_HEADER)
new_file.write(ORIGINAL_SIZE_TEMPLATE % file_info.original_size)
else:
line = old_file.readline()
match = re.match(ORIGINAL_SIZE_REGEX, line)
if match:
original_size_bytes = int(match.group(1))
header_size_bytes = new_file.tell()
# Move old file reader to the beginning of the file.
old_file.seek(0, os.SEEK_SET)
new_file.write(old_file.read(
int((file_size_limit_byte - header_size_bytes) *
HEAD_SIZE_PERCENT)))
# Position to seek from the end of the file.
seek_pos = -(file_size_limit_byte - new_file.tell() -
len(TRIMMED_FILE_INJECT_TEMPLATE))
bytes_to_skip = original_size_bytes + seek_pos - old_file.tell()
# Adjust seek position based on string TRIMMED_FILE_INJECT_TEMPLATE
seek_pos += len(str(bytes_to_skip)) - 2
bytes_to_skip = original_size_bytes + seek_pos - old_file.tell()
new_file.write(TRIMMED_FILE_INJECT_TEMPLATE % bytes_to_skip)
old_file.seek(seek_pos, os.SEEK_END)
new_file.write(old_file.read())
stat = os.stat(file_info.path)
if not throttler_lib.try_delete_file_on_disk(file_info.path):
# Clean up the intermediate file.
throttler_lib.try_delete_file_on_disk(new_path)
utils_lib.LOG('Failed to shrink %s' % file_info.path)
return
os.rename(new_path, file_info.path)
# Modify the new file's timestamp to the old one.
os.utime(file_info.path, (stat.st_atime, stat.st_mtime))
# Update the trimmed_size.
file_info.trimmed_size = file_info.size
def try_delete_file_on_disk(path):
"""Try to delete the give file on disk.
@param path: Path to the file.
@returns: True if the file is deleted, False otherwise.
"""
try:
utils_lib.LOG('Deleting file %s.' % path)
os.remove(path)
return True
except OSError as e:
utils_lib.LOG('Failed to delete file %s, Error: %s' % (path, e))
def _delete_file(file_info):
"""Delete the given file and update the summary.
@param file_info: A ResultInfo object containing summary for the file to be
shrunk.
"""
utils_lib.LOG('Deleting file %s.' % file_info.path)
try:
os.remove(file_info.path)
except OSError as e:
utils_lib.LOG('Failed to delete file %s Error: %s' %
(file_info.path, e))
# Update the trimmed_size in ResultInfo.
file_info.trimmed_size = 0
def execute(path, max_size_KB):
"""Execute the script with given arguments.
@param path: Path to build directory summary.
@param max_size_KB: Maximum result size in KB.
"""
utils_lib.LOG('Running result_tools/utils on path: %s' % path)
if max_size_KB > 0:
utils_lib.LOG('Throttle result size to : %s' %
utils_lib.get_size_string(max_size_KB * 1024))
result_dir = path
if not os.path.isdir(result_dir):
result_dir = os.path.dirname(result_dir)
summary = result_info.ResultInfo.build_from_path(path)
summary_json = json.dumps(summary)
summary_file = get_unique_dir_summary_file(result_dir)
# Make sure there is enough free disk to write the file
stat = os.statvfs(path)
free_space = stat.f_frsize * stat.f_bavail
if free_space - len(summary_json) < MIN_FREE_DISK_BYTES:
raise utils_lib.NotEnoughDiskError(
'Not enough disk space after saving the summary file. '
'Available free disk: %s bytes. Summary file size: %s bytes.' %
(free_space, len(summary_json)))
with open(summary_file, 'w') as f:
f.write(summary_json)
utils_lib.LOG('Directory summary of %s is saved to file %s.' %
(path, summary_file))
if max_size_KB > 0 and summary.trimmed_size > 0:
old_size = summary.trimmed_size
throttle_probability = float(max_size_KB * 1024) / old_size
if random.random() < throttle_probability:
utils_lib.LOG(
'Skip throttling %s: size=%s, throttle_probability=%s' %
(path, old_size, throttle_probability))
else:
_throttle_results(summary, max_size_KB)
if summary.trimmed_size < old_size:
# Files are throttled, save the updated summary file.
utils_lib.LOG('Overwrite the summary file: %s' % summary_file)
result_info.save_summary(summary, summary_file)
def merge_summaries(path):
"""Merge all directory summaries in the given path.
This function calculates the total size of result files being collected for
the test device and the files generated on the drone. It also returns merged
directory summary.
@param path: A path to search for directory summaries.
@return a tuple of (client_collected_bytes, merged_summary, files):
client_collected_bytes: The total size of results collected from
the DUT. The number can be larger than the total file size of
the given path, as files can be overwritten or removed.
merged_summary: The merged directory summary of the given path.
files: All summary files in the given path, including
sub-directories.
"""
path = _preprocess_result_dir_path(path)
# Find all directory summary files and sort them by the time stamp in file
# name.
summary_files = []
for root, _, filenames in os.walk(path):
for filename in fnmatch.filter(filenames, 'dir_summary_*.json'):
summary_files.append(os.path.join(root, filename))
summary_files = sorted(summary_files, key=os.path.getmtime)
all_summaries = []
for summary_file in summary_files:
try:
summary = result_info.load_summary_json_file(summary_file)
summary = _relocate_summary(path, summary_file, summary)
all_summaries.append(summary)
except (IOError, ValueError) as e:
utils_lib.LOG('Failed to load summary file %s Error: %s' %
(summary_file, e))
# Merge all summaries.
merged_summary = all_summaries[0] if len(all_summaries) > 0 else None
for summary in all_summaries[1:]:
merged_summary.merge(summary)
# After all summaries from the test device (client side) are merged, we can
# get the total size of result files being transfered from the test device.
# If there is no directory summary collected, default client_collected_bytes
# to 0.
client_collected_bytes = 0
if merged_summary:
client_collected_bytes = merged_summary.collected_size
# Get the summary of current directory
last_summary = result_info.ResultInfo.build_from_path(path)
if merged_summary:
merged_summary.merge(last_summary, is_final=True)
_delete_missing_entries(merged_summary, last_summary)
else:
merged_summary = last_summary
return client_collected_bytes, merged_summary, summary_files
def check_throttle_limit(summary, max_result_size_KB):
"""Check if throttling is enough already.
@param summary: A ResultInfo object containing result summary.
@param max_result_size_KB: Maximum test result size in KB.
@return: True if the result directory has been trimmed to be smaller than
max_result_size_KB.
"""
if (summary.trimmed_size <= max_result_size_KB * 1024):
utils_lib.LOG('Maximum result size is reached (current result'
'size is %s (limit is %s).' %
(utils_lib.get_size_string(summary.trimmed_size),
utils_lib.get_size_string(max_result_size_KB * 1024)))
return True
else:
return False
def _delete_summaries(path):
"""Delete all directory summary files in the given directory.
This is to cleanup the directory so no summary files are left behind to
affect later tests.
@param path: Path to cleanup directory summary.
"""
# Only summary files directly under the `path` needs to be cleaned.
summary_files = glob.glob(os.path.join(path, SUMMARY_FILE_PATTERN))
for summary in summary_files:
try:
os.remove(summary)
except IOError as e:
utils_lib.LOG('Failed to delete summary: %s. Error: %s' %
(summary, e))
def throttle(summary, max_result_size_KB):
"""Throttle the files in summary by de-duplicating files.
Stop throttling until all files are processed or the result size is already
reduced to be under the given max_result_size_KB.
@param summary: A ResultInfo object containing result summary.
@param max_result_size_KB: Maximum test result size in KB.
"""
_, grouped_files = throttler_lib.sort_result_files(summary)
for pattern in throttler_lib.RESULT_THROTTLE_PRIORITY:
throttable_files = list(
throttler_lib.get_throttleable_files(grouped_files[pattern],
NO_DEDUPE_FILE_PATTERNS))
for info in throttable_files:
info.parent_dir = os.path.dirname(info.path)
info.prefix = re.match(PREFIX_PATTERN, info.name).group(1)
# Group files for each parent directory
grouped_infos = _group_by(throttable_files, ['parent_dir', 'prefix'])
for infos in grouped_infos.values():
if (len(infos) <=
OLDEST_FILES_TO_KEEP_COUNT + NEWEST_FILES_TO_KEEP_COUNT):
# No need to dedupe if the count of file is too few.
continue
# Remove files can be deduped
utils_lib.LOG('De-duplicating files in %s with the same prefix of '
'"%s"' % (infos[0].parent_dir, infos[0].prefix))
#dedupe_file_infos = [i.result_info for i in infos]
_dedupe_files(summary, infos, max_result_size_KB)
if throttler_lib.check_throttle_limit(summary, max_result_size_KB):
return
def _throttle_results(summary, max_result_size_KB):
"""Throttle the test results by limiting to the given maximum size.
@param summary: A ResultInfo object containing result summary.
@param max_result_size_KB: Maximum test result size in KB.
"""
if throttler_lib.check_throttle_limit(summary, max_result_size_KB):
utils_lib.LOG(
'Result size is %s, which is less than %d KB. No need to '
'throttle.' %
(utils_lib.get_size_string(summary.trimmed_size),
max_result_size_KB))
return
args = {'summary': summary,
'max_result_size_KB': max_result_size_KB}
args_skip_autotest_log = copy.copy(args)
args_skip_autotest_log['skip_autotest_log'] = True
# Apply the throttlers in following order.
throttlers = [
(shrink_file_throttler, copy.copy(args_skip_autotest_log)),
(zip_file_throttler, copy.copy(args_skip_autotest_log)),
(shrink_file_throttler, copy.copy(args)),
(dedupe_file_throttler, copy.copy(args)),
(zip_file_throttler, copy.copy(args)),
]
# Add another zip_file_throttler to compress the files being shrunk.
# The threshold is set to half of the DEFAULT_FILE_SIZE_LIMIT_BYTE of
# shrink_file_throttler.
new_args = copy.copy(args)
new_args['file_size_threshold_byte'] = 50 * 1024
throttlers.append((zip_file_throttler, new_args))
# If the above throttlers still can't reduce the result size to be under
# max_result_size_KB, try to delete files with various threshold, starting
# at 5MB then lowering to 100KB.
delete_file_thresholds = [5*1024*1024, 1*1024*1024, 100*1024]
# Try to keep tgz files first.
exclude_file_patterns = ['.*\.tgz']
for threshold in delete_file_thresholds:
new_args = copy.copy(args)
new_args.update({'file_size_threshold_byte': threshold,
'exclude_file_patterns': exclude_file_patterns})
throttlers.append((delete_file_throttler, new_args))
# Add one more delete_file_throttler to not skipping tgz files.
new_args = copy.copy(args)
new_args.update({'file_size_threshold_byte': delete_file_thresholds[-1]})
throttlers.append((delete_file_throttler, new_args))
# Run the throttlers in order until result size is under max_result_size_KB.
old_size = summary.trimmed_size
for throttler, args in throttlers:
try:
args_without_summary = copy.copy(args)
del args_without_summary['summary']
utils_lib.LOG('Applying throttler %s, args: %s' %
(throttler.__name__, args_without_summary))
throttler.throttle(**args)
if throttler_lib.check_throttle_limit(summary, max_result_size_KB):
return
except:
utils_lib.LOG('Failed to apply throttler %s. Exception: %s' %
(throttler, traceback.format_exc()))
finally:
new_size = summary.trimmed_size
if new_size == old_size:
utils_lib.LOG('Result size was not changed: %s.' % old_size)
else:
utils_lib.LOG('Result size was reduced from %s to %s.' %
(utils_lib.get_size_string(old_size),
utils_lib.get_size_string(new_size)))