def extract_items(target_files, target_files_temp_dir, extract_item_list):
"""Extract items from target files to temporary directory.
This function extracts from the specified target files zip archive into the
specified temporary directory, the items specified in the extract item list.
Args:
target_files: The target files zip archive from which to extract items.
target_files_temp_dir: The temporary directory where the extracted items
will land.
extract_item_list: A list of items to extract.
Returns:
On success, 0. Otherwise, a non-zero exit code from unzip.
"""
logger.info('extracting from %s', target_files)
# Filter the extract_item_list to remove any items that do not exist in the
# zip file. Otherwise, the extraction step will fail.
with zipfile.ZipFile(target_files, 'r',
allowZip64=True) as target_files_zipfile:
target_files_namelist = target_files_zipfile.namelist()
filtered_extract_item_list = []
for pattern in extract_item_list:
matching_namelist = fnmatch.filter(target_files_namelist, pattern)
if not matching_namelist:
logger.warning('no match for %s', pattern)
else:
filtered_extract_item_list.append(pattern)
# Extract the filtered_extract_item_list from target_files into
# target_files_temp_dir.
# TODO(b/124464492): Extend common.UnzipTemp() to handle this use case.
command = ['unzip', '-n', '-q', '-d', target_files_temp_dir, target_files
] + filtered_extract_item_list
result = common.RunAndWait(command, verbose=True)
if result != 0:
logger.error('extract_items command %s failed %d', str(command),
result)
return result
return 0
def create_target_files_archive(output_file, source_dir, temp_dir):
"""Creates archive from target package.
Args:
output_file: The name of the zip archive target files package.
source_dir: The target directory contains package to be archived.
temp_dir: Path to temporary directory for any intermediate files.
"""
output_target_files_list = os.path.join(temp_dir, 'output.list')
output_zip = os.path.abspath(output_file)
output_target_files_meta_dir = os.path.join(source_dir, 'META')
def files_from_path(target_path, extra_args=None):
"""Gets files under the given path and return a sorted list."""
find_command = ['find', target_path] + (extra_args or [])
find_process = common.Run(find_command,
stdout=subprocess.PIPE,
verbose=False)
return common.RunAndCheckOutput(['sort'],
stdin=find_process.stdout,
verbose=False)
meta_content = files_from_path(output_target_files_meta_dir)
other_content = files_from_path(
source_dir,
['-path', output_target_files_meta_dir, '-prune', '-o', '-print'])
with open(output_target_files_list, 'w') as f:
f.write(meta_content)
f.write(other_content)
command = [
'soong_zip',
'-d',
'-o',
output_zip,
'-C',
source_dir,
'-l',
output_target_files_list,
]
logger.info('creating %s', output_file)
common.RunAndWait(command, verbose=True)
logger.info('finished creating %s', output_file)
return output_zip
def create_target_files_archive(output_file, source_dir, temp_dir):
"""Creates archive from target package.
Args:
output_file: The name of the zip archive target files package.
source_dir: The target directory contains package to be archived.
temp_dir: Path to temporary directory for any intermediate files.
"""
output_target_files_list = os.path.join(temp_dir, 'output.list')
output_zip = os.path.abspath(output_file)
output_target_files_meta_dir = os.path.join(source_dir, 'META')
meta_content = files_from_path(output_target_files_meta_dir)
other_content = files_from_path(
source_dir,
['-path', output_target_files_meta_dir, '-prune', '-o', '-print'])
with open(output_target_files_list, 'w') as f:
f.write(meta_content)
f.write(other_content)
command = [
'soong_zip',
'-d',
'-o',
output_zip,
'-C',
source_dir,
'-l',
output_target_files_list,
]
logger.info('creating %s', output_file)
common.RunAndWait(command, verbose=True)
logger.info('finished creating %s', output_file)
return output_zip
def merge_target_files(temp_dir, system_target_files, system_item_list,
system_misc_info_keys, other_target_files,
other_item_list, output_target_files, output_dir,
output_item_list, output_ota, output_img,
output_super_empty, rebuild_recovery):
"""Merge two target files packages together.
This function takes system and other target files packages as input, performs
various file extractions, special case processing, and finally creates a
merged zip archive as output.
Args:
temp_dir: The name of a directory we use when we extract items from the
input target files packages, and also a scratch directory that we use for
temporary files.
system_target_files: The name of the zip archive containing the system
partial target files package.
system_item_list: The list of items to extract from the partial system
target files package as is, meaning these items will land in the output
target files package exactly as they appear in the input partial system
target files package.
system_misc_info_keys: The list of keys to obtain from the system instance
of META/misc_info.txt. The remaining keys from the other instance.
other_target_files: The name of the zip archive containing the other partial
target files package.
other_item_list: The list of items to extract from the partial other target
files package as is, meaning these items will land in the output target
files package exactly as they appear in the input partial other target
files package.
output_target_files: The name of the output zip archive target files package
created by merging system and other.
output_dir: The destination directory for saving merged files.
output_item_list: The list of items to copy into the output_dir.
output_ota: The name of the output zip archive ota package.
output_img: The name of the output zip archive img package.
output_super_empty: If provided, creates a super_empty.img file from the
merged target files package and saves it at this path.
rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
devices and write it to the system image.
"""
logger.info('starting: merge system %s and other %s into output %s',
system_target_files, other_target_files, output_target_files)
# Create directory names that we'll use when we extract files from system,
# and other, and for zipping the final output.
system_target_files_temp_dir = os.path.join(temp_dir, 'system')
other_target_files_temp_dir = os.path.join(temp_dir, 'other')
output_target_files_temp_dir = os.path.join(temp_dir, 'output')
# Extract "as is" items from the input system partial target files package.
# We extract them directly into the output temporary directory since the
# items do not need special case processing.
extract_items(target_files=system_target_files,
target_files_temp_dir=output_target_files_temp_dir,
extract_item_list=system_item_list)
# Extract "as is" items from the input other partial target files package. We
# extract them directly into the output temporary directory since the items
# do not need special case processing.
extract_items(target_files=other_target_files,
target_files_temp_dir=output_target_files_temp_dir,
extract_item_list=other_item_list)
# Extract "special" items from the input system partial target files package.
# We extract these items to different directory since they require special
# processing before they will end up in the output directory.
extract_items(target_files=system_target_files,
target_files_temp_dir=system_target_files_temp_dir,
extract_item_list=system_extract_special_item_list)
# Extract "special" items from the input other partial target files package.
# We extract these items to different directory since they require special
# processing before they will end up in the output directory.
extract_items(target_files=other_target_files,
target_files_temp_dir=other_target_files_temp_dir,
extract_item_list=other_extract_special_item_list)
# Now that the temporary directories contain all the extracted files, perform
# special case processing on any items that need it. After this function
# completes successfully, all the files we need to create the output target
# files package are in place.
process_special_cases(
system_target_files_temp_dir=system_target_files_temp_dir,
other_target_files_temp_dir=other_target_files_temp_dir,
output_target_files_temp_dir=output_target_files_temp_dir,
system_misc_info_keys=system_misc_info_keys,
rebuild_recovery=rebuild_recovery)
# Regenerate IMAGES in the temporary directory.
add_img_args = ['--verbose']
if rebuild_recovery:
add_img_args.append('--rebuild_recovery')
add_img_args.append(output_target_files_temp_dir)
add_img_to_target_files.main(add_img_args)
# Create super_empty.img using the merged misc_info.txt.
misc_info_txt = os.path.join(output_target_files_temp_dir, 'META',
'misc_info.txt')
def read_helper():
with open(misc_info_txt) as f:
return list(f.read().splitlines())
use_dynamic_partitions = common.LoadDictionaryFromLines(
read_helper()).get('use_dynamic_partitions')
if use_dynamic_partitions != 'true' and output_super_empty:
raise ValueError(
'Building super_empty.img requires use_dynamic_partitions=true.')
elif use_dynamic_partitions == 'true':
super_empty_img = os.path.join(output_target_files_temp_dir, 'IMAGES',
'super_empty.img')
build_super_image_args = [
misc_info_txt,
super_empty_img,
]
build_super_image.main(build_super_image_args)
# Copy super_empty.img to the user-provided output_super_empty location.
if output_super_empty:
shutil.copyfile(super_empty_img, output_super_empty)
# Create the IMG package from the merged target files (before zipping, in
# order to avoid an unnecessary unzip and copy).
if output_img:
img_from_target_files_args = [
output_target_files_temp_dir,
output_img,
]
img_from_target_files.main(img_from_target_files_args)
# Finally, create the output target files zip archive and/or copy the
# output items to the output target files directory.
if output_dir:
copy_items(output_target_files_temp_dir, output_dir, output_item_list)
if not output_target_files:
return
output_zip = os.path.abspath(output_target_files)
output_target_files_list = os.path.join(temp_dir, 'output.list')
output_target_files_meta_dir = os.path.join(output_target_files_temp_dir,
'META')
find_command = [
'find',
output_target_files_meta_dir,
]
find_process = common.Run(find_command,
stdout=subprocess.PIPE,
verbose=False)
meta_content = common.RunAndCheckOutput(['sort'],
stdin=find_process.stdout,
verbose=False)
find_command = [
'find', output_target_files_temp_dir, '-path',
output_target_files_meta_dir, '-prune', '-o', '-print'
]
find_process = common.Run(find_command,
stdout=subprocess.PIPE,
verbose=False)
other_content = common.RunAndCheckOutput(['sort'],
stdin=find_process.stdout,
verbose=False)
with open(output_target_files_list, 'wb') as f:
f.write(meta_content)
f.write(other_content)
command = [
'soong_zip',
'-d',
'-o',
output_zip,
'-C',
output_target_files_temp_dir,
'-l',
output_target_files_list,
]
logger.info('creating %s', output_target_files)
common.RunAndWait(command, verbose=True)
# Create the OTA package from the merged target files package.
if output_ota:
ota_from_target_files_args = [
output_zip,
output_ota,
]
ota_from_target_files.main(ota_from_target_files_args)
def merge_target_files(temp_dir, system_target_files, other_target_files,
output_target_files):
"""Merge two target files packages together.
This function takes system and other target files packages as input, performs
various file extractions, special case processing, and finally creates a
merged zip archive as output.
Args:
temp_dir: The name of a directory we use when we extract items from the
input target files packages, and also a scratch directory that we use for
temporary files.
system_target_files: The name of the zip archive containing the system
partial target files package.
other_target_files: The name of the zip archive containing the other
partial target files package.
output_target_files: The name of the output zip archive target files
package created by merging system and other.
Returns:
On success, 0. Otherwise, a non-zero exit code.
"""
# Create directory names that we'll use when we extract files from system,
# and other, and for zipping the final output.
system_target_files_temp_dir = os.path.join(temp_dir, 'system')
other_target_files_temp_dir = os.path.join(temp_dir, 'other')
output_target_files_temp_dir = os.path.join(temp_dir, 'output')
# Extract "as is" items from the input system partial target files package.
# We extract them directly into the output temporary directory since the
# items do not need special case processing.
result = extract_items(target_files=system_target_files,
target_files_temp_dir=output_target_files_temp_dir,
extract_item_list=system_extract_as_is_item_list)
if result != 0:
return result
# Extract "as is" items from the input other partial target files package. We
# extract them directly into the output temporary directory since the items
# do not need special case processing.
result = extract_items(target_files=other_target_files,
target_files_temp_dir=output_target_files_temp_dir,
extract_item_list=other_extract_as_is_item_list)
if result != 0:
return result
# Extract "special" items from the input system partial target files package.
# We extract these items to different directory since they require special
# processing before they will end up in the output directory.
result = extract_items(target_files=system_target_files,
target_files_temp_dir=system_target_files_temp_dir,
extract_item_list=system_extract_special_item_list)
if result != 0:
return result
# Extract "special" items from the input other partial target files package.
# We extract these items to different directory since they require special
# processing before they will end up in the output directory.
result = extract_items(target_files=other_target_files,
target_files_temp_dir=other_target_files_temp_dir,
extract_item_list=other_extract_special_item_list)
if result != 0:
return result
# Now that the temporary directories contain all the extracted files, perform
# special case processing on any items that need it. After this function
# completes successfully, all the files we need to create the output target
# files package are in place.
result = process_special_cases(
temp_dir=temp_dir,
system_target_files_temp_dir=system_target_files_temp_dir,
other_target_files_temp_dir=other_target_files_temp_dir,
output_target_files_temp_dir=output_target_files_temp_dir)
if result != 0:
return result
# Regenerate IMAGES in the temporary directory.
add_img_args = [
'--verbose',
output_target_files_temp_dir,
]
add_img_to_target_files.main(add_img_args)
# Finally, create the output target files zip archive.
output_zip = os.path.abspath(output_target_files)
output_target_files_list = os.path.join(temp_dir, 'output.list')
output_target_files_meta_dir = os.path.join(output_target_files_temp_dir,
'META')
command = [
'find',
output_target_files_meta_dir,
]
# TODO(bpeckham): sort this to be more like build.
meta_content = common.RunAndCheckOutput(command, verbose=False)
command = [
'find', output_target_files_temp_dir, '-path',
output_target_files_meta_dir, '-prune', '-o', '-print'
]
# TODO(bpeckham): sort this to be more like build.
other_content = common.RunAndCheckOutput(command, verbose=False)
with open(output_target_files_list, 'wb') as f:
f.write(meta_content)
f.write(other_content)
command = [
'soong_zip',
'-d',
'-o',
output_zip,
'-C',
output_target_files_temp_dir,
'-l',
output_target_files_list,
]
logger.info('creating %s', output_target_files)
result = common.RunAndWait(command, verbose=True)
if result != 0:
logger.error('zip command %s failed %d', str(command), result)
return result
return 0
def process_file_contexts_bin(temp_dir, output_target_files_temp_dir):
"""Perform special processing for META/file_contexts.bin.
This function combines plat_file_contexts and vendor_file_contexts, which are
expected to already be extracted in temp_dir, to produce a merged
file_contexts.bin that will land in temp_dir at META/file_contexts.bin.
Args:
temp_dir: The name of a scratch directory that this function can use for
intermediate files generated during processing.
output_target_files_temp_dir: The name of the working directory that must
already contain plat_file_contexts and vendor_file_contexts (in the
appropriate sub directories), and to which META/file_contexts.bin will be
written.
Returns:
On success, 0. Otherwise, a non-zero exit code.
"""
# To create a merged file_contexts.bin file, we use the system and vendor
# file contexts files as input, the m4 tool to combine them, the sorting tool
# to sort, and finally the sefcontext_compile tool to generate the final
# output. We currently omit a checkfc step since the files had been checked
# as part of the build.
# The m4 step concatenates the two input files contexts files. Since m4
# writes to stdout, we receive that into an array of bytes, and then write it
# to a file.
# Collect the file contexts that we're going to combine from SYSTEM, VENDOR,
# PRODUCT, and ODM. We require SYSTEM and VENDOR, but others are optional.
file_contexts_list = []
for partition in ['SYSTEM', 'VENDOR', 'PRODUCT', 'ODM']:
prefix = 'plat' if partition == 'SYSTEM' else partition.lower()
file_contexts = os.path.join(output_target_files_temp_dir, partition,
'etc', 'selinux',
prefix + '_file_contexts')
mandatory = partition in ['SYSTEM', 'VENDOR']
if mandatory or os.path.isfile(file_contexts):
file_contexts_list.append(file_contexts)
else:
logger.warning('file not found: %s', file_contexts)
command = ['m4', '--fatal-warnings', '-s'] + file_contexts_list
merged_content = common.RunAndCheckOutput(command, verbose=False)
merged_file_contexts_txt = os.path.join(temp_dir,
'merged_file_contexts.txt')
with open(merged_file_contexts_txt, 'wb') as f:
f.write(merged_content)
# The sort step sorts the concatenated file.
sorted_file_contexts_txt = os.path.join(temp_dir,
'sorted_file_contexts.txt')
command = ['fc_sort', merged_file_contexts_txt, sorted_file_contexts_txt]
# TODO(b/124521133): Refector RunAndWait to raise exception on failure.
result = common.RunAndWait(command, verbose=True)
if result != 0:
return result
# Finally, the compile step creates the final META/file_contexts.bin.
file_contexts_bin = os.path.join(output_target_files_temp_dir, 'META',
'file_contexts.bin')
command = [
'sefcontext_compile',
'-o',
file_contexts_bin,
sorted_file_contexts_txt,
]
result = common.RunAndWait(command, verbose=True)
if result != 0:
return result
return 0
def merge_target_files(temp_dir, system_target_files, system_item_list,
system_misc_info_keys, other_target_files,
other_item_list, output_target_files, rebuild_recovery):
"""Merge two target files packages together.
This function takes system and other target files packages as input, performs
various file extractions, special case processing, and finally creates a
merged zip archive as output.
Args:
temp_dir: The name of a directory we use when we extract items from the
input target files packages, and also a scratch directory that we use for
temporary files.
system_target_files: The name of the zip archive containing the system
partial target files package.
system_item_list: The list of items to extract from the partial system
target files package as is, meaning these items will land in the output
target files package exactly as they appear in the input partial system
target files package.
system_misc_info_keys: The list of keys to obtain from the system instance
of META/misc_info.txt. The remaining keys from the other instance.
other_target_files: The name of the zip archive containing the other
partial target files package.
other_item_list: The list of items to extract from the partial other
target files package as is, meaning these items will land in the output
target files package exactly as they appear in the input partial other
target files package.
output_target_files: The name of the output zip archive target files
package created by merging system and other.
rebuild_recovery: If true, rebuild the recovery patch used by non-A/B
devices and write it to the system image.
"""
logger.info('starting: merge system %s and other %s into output %s',
system_target_files, other_target_files, output_target_files)
# Create directory names that we'll use when we extract files from system,
# and other, and for zipping the final output.
system_target_files_temp_dir = os.path.join(temp_dir, 'system')
other_target_files_temp_dir = os.path.join(temp_dir, 'other')
output_target_files_temp_dir = os.path.join(temp_dir, 'output')
# Extract "as is" items from the input system partial target files package.
# We extract them directly into the output temporary directory since the
# items do not need special case processing.
extract_items(target_files=system_target_files,
target_files_temp_dir=output_target_files_temp_dir,
extract_item_list=system_item_list)
# Extract "as is" items from the input other partial target files package. We
# extract them directly into the output temporary directory since the items
# do not need special case processing.
extract_items(target_files=other_target_files,
target_files_temp_dir=output_target_files_temp_dir,
extract_item_list=other_item_list)
# Extract "special" items from the input system partial target files package.
# We extract these items to different directory since they require special
# processing before they will end up in the output directory.
extract_items(target_files=system_target_files,
target_files_temp_dir=system_target_files_temp_dir,
extract_item_list=system_extract_special_item_list)
# Extract "special" items from the input other partial target files package.
# We extract these items to different directory since they require special
# processing before they will end up in the output directory.
extract_items(target_files=other_target_files,
target_files_temp_dir=other_target_files_temp_dir,
extract_item_list=other_extract_special_item_list)
# Now that the temporary directories contain all the extracted files, perform
# special case processing on any items that need it. After this function
# completes successfully, all the files we need to create the output target
# files package are in place.
process_special_cases(
temp_dir=temp_dir,
system_target_files_temp_dir=system_target_files_temp_dir,
other_target_files_temp_dir=other_target_files_temp_dir,
output_target_files_temp_dir=output_target_files_temp_dir,
system_misc_info_keys=system_misc_info_keys,
rebuild_recovery=rebuild_recovery)
# Regenerate IMAGES in the temporary directory.
add_img_args = ['--verbose']
if rebuild_recovery:
add_img_args.append('--rebuild_recovery')
add_img_args.append(output_target_files_temp_dir)
add_img_to_target_files.main(add_img_args)
# Finally, create the output target files zip archive.
output_zip = os.path.abspath(output_target_files)
output_target_files_list = os.path.join(temp_dir, 'output.list')
output_target_files_meta_dir = os.path.join(output_target_files_temp_dir,
'META')
command = [
'find',
output_target_files_meta_dir,
]
# TODO(bpeckham): sort this to be more like build.
meta_content = common.RunAndCheckOutput(command, verbose=False)
command = [
'find', output_target_files_temp_dir, '-path',
output_target_files_meta_dir, '-prune', '-o', '-print'
]
# TODO(bpeckham): sort this to be more like build.
other_content = common.RunAndCheckOutput(command, verbose=False)
with open(output_target_files_list, 'wb') as f:
f.write(meta_content)
f.write(other_content)
command = [
'soong_zip',
'-d',
'-o',
output_zip,
'-C',
output_target_files_temp_dir,
'-l',
output_target_files_list,
]
logger.info('creating %s', output_target_files)
common.RunAndWait(command, verbose=True)
