def _configure_build_options():
if OPTIONS.parse(sys.argv[1:]):
print 'Args error'
return False
# Write out the configure file early so all other scripts can use
# the options passed into configure. (e.g., sync_chrome).
OPTIONS.write_configure_file()
# Target directory is replaced. If an old directory, out/target/<target>,
# exists, move it to the new place, out/target/<target>_<opt>.
old_path = os.path.join('out/target', OPTIONS.target())
new_path = build_common.get_build_dir()
if os.path.lexists(old_path):
if os.path.isdir(old_path) and not os.path.islink(old_path):
if os.path.exists(new_path):
file_util.rmtree(old_path)
else:
shutil.move(old_path, new_path)
else:
os.remove(old_path)
# Create an empty directory as a placeholder if necessary.
file_util.makedirs_safely(new_path)
# Create a symlink from new place to old place to keep as compatible as
# possible.
os.symlink(os.path.basename(new_path), old_path)
# Write out the configure file to a target specific location, which can be
# queried later to find out what the config for a target was.
OPTIONS.write_configure_file(build_common.get_target_configure_options_file())
OPTIONS.set_up_goma()
return True
def _download_adb_source(force):
"""Downloads the adb source code with git, if needed.
The source tree will be placed at out/adb/src.
"""
source_dir = os.path.join(ADB_OUTPUT_DIR, "src")
stamp_file_path = os.path.join(source_dir, "STAMP")
stamp_file = build_common.StampFile(ADB_SOURCE_VERSION, stamp_file_path, force=force)
if stamp_file.is_up_to_date():
return
if os.path.exists(source_dir):
file_util.rmtree(source_dir)
try:
timer = build_common.SimpleTimer()
timer.start("Downloading the adb source code", show=True)
_run_git_clone(BRANCH, SYSTEM_CORE_URL, os.path.join(ADB_OUTPUT_DIR, "src/system/core"))
_run_git_clone(BRANCH, ZLIB_URL, os.path.join(ADB_OUTPUT_DIR, "src/external/zlib"))
timer.done()
except Exception as exception:
print exception
raise Exception("Failed to download the adb source code")
stamp_file.update()
def _download_adb_source(force):
"""Downloads the adb source code with git, if needed.
The source tree will be placed at out/adb/src.
"""
source_dir = os.path.join(ADB_OUTPUT_DIR, 'src')
stamp_file_path = os.path.join(source_dir, 'STAMP')
stamp_file = build_common.StampFile(ADB_SOURCE_VERSION,
stamp_file_path,
force=force)
if stamp_file.is_up_to_date():
return
if os.path.exists(source_dir):
file_util.rmtree(source_dir)
try:
timer = build_common.SimpleTimer()
timer.start('Downloading the adb source code', show=True)
_run_git_clone(BRANCH, SYSTEM_CORE_URL,
os.path.join(ADB_OUTPUT_DIR, 'src/system/core'))
_run_git_clone(BRANCH, ZLIB_URL,
os.path.join(ADB_OUTPUT_DIR, 'src/external/zlib'))
timer.done()
except Exception as exception:
print exception
raise Exception('Failed to download the adb source code')
stamp_file.update()
def _build_apk(source_path, use_ndk, use_clang, build_path, install_apk, debug,
verbose):
if not os.path.isdir(_SDK_PATH):
raise Exception('Missing SDK path: ' + str(_SDK_PATH))
print
print '--------------------------'
print 'Building ' + os.path.basename(install_apk)
print '--------------------------'
# We use this work directory in order to allow us to completely
# create it from scratch every time we build. We cannot do that
# to the build_path since files in there (like the build.log)
# should not be deleted on every run.
work_path = os.path.join(build_path, 'work')
if os.path.isdir(work_path):
file_util.rmtree(work_path)
shutil.copytree(os.path.join('.', source_path), work_path)
print os.path.join(_SDK_PATH, 'tools', 'android')
# Any target 14+ should work (tested on 21).
subprocess.check_call([
os.path.join(_SDK_PATH, 'tools',
'android'), 'update', 'project', '--target',
'android-%d' % toolchain.get_android_api_level(), '--path', '.',
'--name', 'test_app'
],
cwd=work_path)
if use_ndk:
if not os.path.isdir(_NDK_PATH):
raise Exception('Missing NDK path: ' + str(_NDK_PATH))
app_optim = 'release'
if debug:
app_optim = 'debug'
if not os.path.exists(os.path.join(work_path, 'jni',
'Application.mk')):
# Write desired ABI before calling ndk-build. Do not drop 'x86' since
# -t=bi uses x86 NDK binaries by default.
open(os.path.join(work_path, 'jni', 'Application.mk'),
'w').write('APP_ABI := x86 armeabi armeabi-v7a\n' +
('APP_OPTIM := %s\n' % app_optim) +
'APP_STL := stlport_static\n')
extras = []
if verbose:
extras.append('V=1')
if use_clang:
extras.append('NDK_TOOLCHAIN_VERSION=clang')
subprocess.check_call([
os.path.join(_NDK_PATH, 'ndk-build'), '-j', '16', '-l', '16',
'ARC_ROOT=' + _ARC_ROOT
] + extras,
cwd=work_path)
subprocess.check_call(
[os.path.join(_TOOLS_ROOT, 'ant', 'bin', 'ant'), 'debug'],
cwd=work_path)
shutil.copyfile(os.path.join(work_path, 'bin', 'test_app-debug.apk'),
install_apk)
def _download_adb(target, force):
"""Downloads the adb executable for Windows or Mac, if needed.
The downloaded executable will be placed at out/adb/win-x86_64/adb.exe or
out/adb/mac-x86_64/adb.
"""
# URL looks like 'https://dl.google.com/android/adt/adt-xxx.zip'
url = DEVTOOLS_URLS[target]
output_dir = os.path.join(ADB_OUTPUT_DIR, target)
stamp_file_path = os.path.join(output_dir, "STAMP")
stamp_file = build_common.StampFile(url, stamp_file_path, force=force) # Use URL as the version.
if stamp_file.is_up_to_date():
return
if os.path.exists(output_dir):
file_util.rmtree(output_dir)
os.makedirs(output_dir)
is_windows = target.startswith("win-")
adb_base_name = "adb.exe" if is_windows else "adb"
# The output file name looks like 'out/adb/win-x86_64/adb.exe'
adb_output_file_name = os.path.join(output_dir, adb_base_name)
zip_file_name = os.path.basename(urlparse.urlparse(url).path)
zip_name = os.path.splitext(zip_file_name)[0]
# The adb path in zip looks like 'adt-xxx/sdk/platform-tools/adb.exe'
adb_path_in_zip = os.path.join(zip_name, "sdk/platform-tools", adb_base_name)
# For Windows, AdbWinApi.dll is also needed.
if is_windows:
dll_path_in_zip = os.path.join(zip_name, "sdk/platform-tools/AdbWinApi.dll")
dll_output_file_name = os.path.join(output_dir, "AdbWinApi.dll")
try:
timer = build_common.SimpleTimer()
timer.start("Downloading the adb executable for %s" % target, show=True)
with contextlib.closing(urllib2.urlopen(url)) as stream, (
zipfile.ZipFile(cStringIO.StringIO(stream.read()))
) as zip_archive:
with open(adb_output_file_name, "w") as adb_file:
# Don't use zipfile.extract() as it creates sub directories.
content = zip_archive.read(adb_path_in_zip)
adb_file.write(content)
os.chmod(adb_output_file_name, stat.S_IRWXU)
# Also extract AdbWinApi.dll for Windows.
if is_windows:
with open(dll_output_file_name, "w") as dll_file:
content = zip_archive.read(dll_path_in_zip)
dll_file.write(content)
os.chmod(dll_output_file_name, stat.S_IRWXU)
timer.done()
except Exception as exception:
print exception
raise Exception("Failed to download the adb executable")
stamp_file.update()
def check_and_perform_update(self):
"""Checks the current and dependency stamps, and performs the update if
they are different."""
start = time.time()
with _persisted_cache_history(self._name, self._cache_base_path,
self._cache_history_size) as history:
# Maintain a recent used history of entries for this path.
history.ensure_recent(self._unpacked_cache_path)
# Temporarily populate the cache path from the final path, if we do not
# already have the contents cached.
self._populate_cache_from_non_symlinked_files(history)
logging.info('%s: Checking %s', self._name,
self._unpacked_final_path)
stamp_file = build_common.StampFile(
self._get_stampfile_content(),
os.path.join(self._unpacked_final_path, 'URL'))
if stamp_file.is_up_to_date():
logging.info('%s: %s is up to date', self._name,
self._unpacked_final_path)
return
logging.info('%s: %s is out of date', self._name,
self._unpacked_final_path)
file_util.rmtree(self._unpacked_final_path, ignore_errors=True)
cached_stamp_file = build_common.StampFile(
self._get_stampfile_content(),
os.path.join(self.unpacked_linked_cache_path, 'URL'))
if not cached_stamp_file.is_up_to_date():
self._fetch_and_cache_package()
# We do this now so that the post_update_work step can run out of
# FINAL_DIR if it wants to.
self.populate_final_directory()
# Do any extra work needed after unpacking the package.
self.post_update_work()
# Write out the updated stamp file
cached_stamp_file.update()
# Reset the mtime on all the entries in the cache.
self.touch_all_files_in_cache()
# Ensure the final directory properly links to the cache.
self.populate_final_directory()
total_time = time.time() - start
if total_time > 1:
print '%s update took %0.3fs' % (self._name[:-5]
if self._name.endswith('Files')
else self._name, total_time)
logging.info('%s: Done. [%0.3fs]', self._name, total_time)
def check_and_perform_update(self):
"""Checks the current and dependency stamps, and performs the update if
they are different."""
start = time.time()
with _persisted_cache_history(self._name, self._cache_base_path,
self._cache_history_size) as history:
# Maintain a recent used history of entries for this path.
history.ensure_recent(self._unpacked_cache_path)
# Temporarily populate the cache path from the final path, if we do not
# already have the contents cached.
self._populate_cache_from_non_symlinked_files(history)
logging.info('%s: Checking %s', self._name, self._unpacked_final_path)
stamp_file = build_common.StampFile(
self._get_stampfile_content(),
os.path.join(self._unpacked_final_path, 'URL'))
if stamp_file.is_up_to_date():
logging.info('%s: %s is up to date', self._name,
self._unpacked_final_path)
return
logging.info('%s: %s is out of date', self._name,
self._unpacked_final_path)
file_util.rmtree(self._unpacked_final_path, ignore_errors=True)
cached_stamp_file = build_common.StampFile(
self._get_stampfile_content(),
os.path.join(self.unpacked_linked_cache_path, 'URL'))
if not cached_stamp_file.is_up_to_date():
self._fetch_and_cache_package()
# We do this now so that the post_update_work step can run out of
# FINAL_DIR if it wants to.
self.populate_final_directory()
# Do any extra work needed after unpacking the package.
self.post_update_work()
# Write out the updated stamp file
cached_stamp_file.update()
# Reset the mtime on all the entries in the cache.
self.touch_all_files_in_cache()
# Ensure the final directory properly links to the cache.
self.populate_final_directory()
total_time = time.time() - start
if total_time > 1:
print '%s update took %0.3fs' % (
self._name[:-5] if self._name.endswith('Files') else self._name,
total_time)
logging.info('%s: Done. [%0.3fs]', self._name, total_time)
def _build_apk(source_path, use_ndk, use_clang, build_path,
install_apk, debug, verbose):
if not os.path.isdir(_SDK_PATH):
raise Exception('Missing SDK path: ' + str(_SDK_PATH))
print
print '--------------------------'
print 'Building ' + os.path.basename(install_apk)
print '--------------------------'
# We use this work directory in order to allow us to completely
# create it from scratch every time we build. We cannot do that
# to the build_path since files in there (like the build.log)
# should not be deleted on every run.
work_path = os.path.join(build_path, 'work')
if os.path.isdir(work_path):
file_util.rmtree(work_path)
shutil.copytree(os.path.join('.', source_path), work_path)
print os.path.join(_SDK_PATH, 'tools', 'android')
# Any target 14+ should work (tested on 21).
subprocess.check_call([
os.path.join(_SDK_PATH, 'tools', 'android'),
'update', 'project',
'--target', 'android-%d' % toolchain.get_android_api_level(),
'--path', '.', '--name', 'test_app'], cwd=work_path)
if use_ndk:
if not os.path.isdir(_NDK_PATH):
raise Exception('Missing NDK path: ' + str(_NDK_PATH))
app_optim = 'release'
if debug:
app_optim = 'debug'
if not os.path.exists(os.path.join(work_path, 'jni', 'Application.mk')):
# Write desired ABI before calling ndk-build. Do not drop 'x86' since
# -t=bi uses x86 NDK binaries by default.
open(os.path.join(work_path, 'jni', 'Application.mk'), 'w').write(
'APP_ABI := x86 armeabi armeabi-v7a\n' +
('APP_OPTIM := %s\n' % app_optim) +
'APP_STL := stlport_static\n')
extras = []
if verbose:
extras.append('V=1')
if use_clang:
extras.append('NDK_TOOLCHAIN_VERSION=clang')
subprocess.check_call([os.path.join(_NDK_PATH, 'ndk-build'),
'-j', '16', '-l', '16',
'ARC_ROOT=' + _ARC_ROOT] + extras,
cwd=work_path)
subprocess.check_call([os.path.join(_TOOLS_ROOT, 'ant', 'bin', 'ant'),
'debug'], cwd=work_path)
shutil.copyfile(os.path.join(work_path, 'bin', 'test_app-debug.apk'),
install_apk)
def _copy_off(where):
if os.path.exists(where):
file_util.rmtree(where)
root = build_common.get_arc_root()
generated_ninja_dir = os.path.join(root, build_common.OUT_DIR,
'generated_ninja')
top_level_ninja = os.path.join(root, 'build.ninja')
if (not os.path.exists(top_level_ninja) or
not os.path.exists(generated_ninja_dir)):
sys.exit('You must run configure first')
shutil.copytree(generated_ninja_dir, where)
shutil.copy(top_level_ninja, where)
def _copy_off(where):
if os.path.exists(where):
file_util.rmtree(where)
root = build_common.get_arc_root()
generated_ninja_dir = os.path.join(root, build_common.OUT_DIR,
'generated_ninja')
top_level_ninja = os.path.join(root, 'build.ninja')
if (not os.path.exists(top_level_ninja)
or not os.path.exists(generated_ninja_dir)):
sys.exit('You must run configure first')
shutil.copytree(generated_ninja_dir, where)
shutil.copy(top_level_ninja, where)
def main():
OPTIONS.parse_configure_file()
parser = argparse.ArgumentParser(
description='Apply dex2oat to sub apks contained as assets in GmsCore.')
parser.add_argument('--input', required=True, help='input apk')
parser.add_argument('--output', required=True, help='output apk')
args = parser.parse_args()
work_dir = tempfile.mkdtemp()
try:
return 0 if _preoptimize_subapk(args.input, args.output, work_dir) else 1
finally:
file_util.rmtree(work_dir)
def _remove_ndk_libraries(apk_path):
"""Remove ndk libraries installed by previous launches.
Package Manager installs shared libraries that match ABI but it doesn't
remove them from previous installation. If apk does not contain the library
for current ABI, installer does not produce an error. In this case
application may launch successfully using previously installed library. We
want to see an error instead.
"""
apk_name = os.path.splitext(os.path.basename(apk_path))[0]
if apk_name:
native_library_directory = os.path.join(build_common.get_arc_root(),
build_common.get_android_root(),
'data', 'app-lib',
apk_name)
file_util.rmtree(native_library_directory, ignore_errors=True)
def setUp(self):
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
atexit.register(lambda: file_util.rmtree(tmpdir, ignore_errors=True))
os.makedirs('foo/bar/baz')
_touch('foo/bar/baz/hoge.cc')
_touch('foo/bar/baz/fuga.py')
_reset_timestamp()
def setUp(self):
tmpdir = tempfile.mkdtemp()
os.chdir(tmpdir)
atexit.register(lambda: file_util.rmtree(tmpdir, ignore_errors=True))
os.makedirs('foo/bar/baz')
_touch('foo/bar/baz/hoge.cc')
_touch('foo/bar/baz/fuga.py')
_reset_timestamp()
def process(target_path_list, ignore_file=None, output_file=None):
target_file_list = _expand_path_list(target_path_list)
ignore_rule = _read_ignore_rule(ignore_file)
target_file_list = _filter_files(target_file_list)
# Create a temporary directory as the output dir of the analyze_diffs.py,
# iff |output_file| is specified.
output_dir = tempfile.mkdtemp(dir='out') if output_file else None
try:
if not _run_lint(target_file_list, ignore_rule, output_dir):
return 1
if output_file:
statistic_list = _process_analyze_diffs_output(output_dir)
with open(output_file, 'wb') as stream:
cPickle.dump(statistic_list, stream)
finally:
if output_dir:
file_util.rmtree(output_dir)
return 0
def process(target_path_list, ignore_file=None, output_file=None):
target_file_list = _expand_path_list(target_path_list)
ignore_rule = _read_ignore_rule(ignore_file)
target_file_list = _filter_files(target_file_list)
# Create a temporary directory as the output dir of the analyze_diffs.py,
# iff |output_file| is specified.
output_dir = tempfile.mkdtemp(dir="out") if output_file else None
try:
if not _run_lint(target_file_list, ignore_rule, output_dir):
return 1
if output_file:
statistic_list = _process_analyze_diffs_output(output_dir)
with open(output_file, "wb") as stream:
cPickle.dump(statistic_list, stream)
finally:
if output_dir:
file_util.rmtree(output_dir)
return 0
def _fetch_and_cache_package(self):
"""Downloads an update file to a temp directory, and manages replacing the
final directory with the stage directory contents."""
try:
# Clean out the cache unpack location.
logging.info('%s: Cleaning %s', self._name,
self._unpacked_cache_path)
file_util.rmtree(self._unpacked_cache_path, ignore_errors=True)
file_util.makedirs_safely(self._unpacked_cache_path)
# Setup the temporary location for the download.
tmp_dir = tempfile.mkdtemp()
try:
downloaded_package_path = os.path.join(tmp_dir, self._name)
# Download the package.
logging.info('%s: Downloading %s', self._name,
downloaded_package_path)
self._download_package_with_retries(self._url,
downloaded_package_path)
# Unpack it.
logging.info('%s: Unpacking %s to %s', self._name,
downloaded_package_path,
self._unpacked_cache_path)
self._unpack_method(downloaded_package_path,
self._unpacked_cache_path)
finally:
file_util.rmtree(tmp_dir, ignore_errors=True)
except:
file_util.rmtree(self._unpacked_cache_path, ignore_errors=True)
raise
def _fetch_and_cache_package(self):
"""Downloads an update file to a temp directory, and manages replacing the
final directory with the stage directory contents."""
try:
# Clean out the cache unpack location.
logging.info('%s: Cleaning %s', self._name, self._unpacked_cache_path)
file_util.rmtree(self._unpacked_cache_path, ignore_errors=True)
file_util.makedirs_safely(self._unpacked_cache_path)
# Setup the temporary location for the download.
tmp_dir = tempfile.mkdtemp()
try:
downloaded_package_path = os.path.join(tmp_dir, self._name)
# Download the package.
logging.info('%s: Downloading %s', self._name,
downloaded_package_path)
self._download_package_with_retries(self._url, downloaded_package_path)
# Unpack it.
logging.info('%s: Unpacking %s to %s', self._name,
downloaded_package_path, self._unpacked_cache_path)
self._unpack_method(downloaded_package_path,
self._unpacked_cache_path)
finally:
file_util.rmtree(tmp_dir, ignore_errors=True)
except:
file_util.rmtree(self._unpacked_cache_path, ignore_errors=True)
raise
def _configure_build_options():
if OPTIONS.parse(sys.argv[1:]):
print 'Args error'
return False
# Write out the configure file early so all other scripts can use
# the options passed into configure. (e.g., sync_chrome).
OPTIONS.write_configure_file()
# Target directory is replaced. If an old directory, out/target/<target>,
# exists, move it to the new place, out/target/<target>_<opt>.
old_path = os.path.join('out/target', OPTIONS.target())
new_path = build_common.get_build_dir()
if os.path.lexists(old_path):
if os.path.isdir(old_path) and not os.path.islink(old_path):
if os.path.exists(new_path):
file_util.rmtree(old_path)
else:
shutil.move(old_path, new_path)
else:
os.remove(old_path)
# Create an empty directory as a placeholder if necessary.
file_util.makedirs_safely(new_path)
# Create a symlink from new place to old place to keep as compatible as
# possible.
os.symlink(os.path.basename(new_path), old_path)
# Write out the configure file to a target specific location, which can be
# queried later to find out what the config for a target was.
OPTIONS.write_configure_file(
build_common.get_target_configure_options_file())
OPTIONS.set_up_goma()
return True
def _set_up_chromium_org_submodules():
CHROMIUM_ORG_ROOT = 'third_party/android/external/chromium_org'
# android/external/chromium_org contains these required submodules. It is not
# posible to have submodules within a submodule path (i.e., chromium_org)
# using git submodules. This is the list of subdirectories relative to
# chromium_org that we need to symlink to the appropriate submodules.
submodules = [
'sdch/open-vcdiff', 'testing/gtest', 'third_party/WebKit',
'third_party/angle', 'third_party/brotli/src',
('third_party/eyesfree/src/android/java/'
'src/com/googlecode/eyesfree/braille'), 'third_party/freetype',
'third_party/icu', 'third_party/leveldatabase/src',
'third_party/libjingle/source/talk',
'third_party/libphonenumber/src/phonenumbers',
'third_party/libphonenumber/src/resources', 'third_party/libsrtp',
'third_party/libvpx', 'third_party/libyuv', 'third_party/mesa/src',
'third_party/openmax_dl', 'third_party/openssl',
'third_party/opus/src', 'third_party/ots',
'third_party/sfntly/cpp/src', 'third_party/skia',
'third_party/smhasher/src', 'third_party/usrsctp/usrsctplib',
'third_party/webrtc', 'third_party/yasm/source/patched-yasm', 'v8'
]
# First remove all existing symlinks to make sure no stale links exist.
for dirpath, dirs, fnames in os.walk(CHROMIUM_ORG_ROOT):
# We only create symlinks for directories.
for name in dirs:
directory = os.path.join(dirpath, name)
if os.path.islink(directory):
os.unlink(directory)
for s in submodules:
target = os.path.join(CHROMIUM_ORG_ROOT, s)
# As an example, this maps 'sdch/open-vcdiff' to
# 'android/external/chromium_org__sdch_open-vcdiff', which is the true
# location of the submodule checkout.
source = 'third_party/android/external/chromium_org__' + s.replace(
'/', '_')
if not os.path.exists(source):
print 'ERROR: path "%s" does not exist.' % source
print 'ERROR: Did you forget to run git submodules update --init?'
sys.exit(1)
# Remove existing symlink for transition from previous version.
# The previous configuration script creates symlinks to the top of
# chromium.org submodules, and now it tries to symlink them one-level
# deeper.
#
# TODO(tzik): Remove this clean up code after the transition is no longer
# needed.
if os.path.islink(target):
os.remove(target)
target_contents = set()
try:
target_contents = set(os.listdir(target))
except OSError as e:
if e.errno != errno.ENOENT:
raise
source_contents = set(os.listdir(source))
for content in source_contents:
link_source = os.path.join(source, content)
link_target = os.path.join(target, content)
# If a real directory exists, remove it explicitly. |overwrite|
# flag does not care for real directories and files, but old symlinks.
if os.path.exists(link_target) and not os.path.islink(link_target):
file_util.rmtree(link_target)
file_util.create_link(link_target, link_source, overwrite=True)
for removed_item in target_contents.difference(source_contents):
os.unlink(os.path.join(target, removed_item))
def setup_output_directory(output_dir):
"""Creates a directory to put all test log files."""
if os.path.exists(output_dir):
file_util.rmtree(output_dir)
file_util.makedirs_safely(output_dir)
def create_staging():
timer = build_common.SimpleTimer()
timer.start('Staging source files', True)
staging_root = build_common.get_staging_root()
# Store where all the old staging links pointed so we can compare after.
old_staging_links = _get_link_targets(staging_root)
if os.path.lexists(staging_root):
file_util.rmtree(staging_root)
_create_symlink_tree(_MODS_DIR, _THIRD_PARTY_DIR, staging_root)
# internal/ is an optional checkout
if build_options.OPTIONS.internal_apks_source_is_internal():
assert build_common.has_internal_checkout()
for name in os.listdir(_INTERNAL_THIRD_PARTY_PATH):
if os.path.exists(os.path.join(_THIRD_PARTY_DIR, name)):
raise Exception(
'Name conflict between internal/third_party and '
'third_party: ' + name)
_create_symlink_tree(_INTERNAL_MODS_PATH, _INTERNAL_THIRD_PARTY_PATH,
staging_root)
subprocess.check_call('internal/build/fix_staging.py')
# src/ is not overlaid on any directory.
_create_symlink_tree(_SRC_DIR, None, os.path.join(staging_root, 'src'))
# Update modification time for files that do not point to the same location
# that they pointed to in the previous tree to make sure they are built.
if old_staging_links:
new_staging_links = _get_link_targets(staging_root)
# Every file (not directory) under staging is in either one of following
# two states:
#
# F. The file itself is a symbolic link to a file under third_party or
# mods.
# D. Some ancestor directory is a symbolic link to a directory under
# third_party. (It is important that we do not create symbolic links to
# directories under mods)
#
# Let us say a file falls under "X-Y" case if it was in state X before
# re-staging and now in state Y. For all 4 possible cases, we can check if
# the actual destination of the file changed or not in the following way:
#
# F-F: We can just compare the target of the link.
# F-D, D-F: The target may have changed, but it needs some complicated
# computation to check. We treat them as changed to be conservative.
# D-D: We can leave it as-is since both point third_party.
#
# So we want to visit all files in state F either in old staging or new
# staging. For this purpose we can iterate through |*_staging_links| as
# they contain all files in state F.
#
# Note that |*_staging_links| may contain directory symbolic links, but
# it is okay to visit them too because directory timestamps do not matter.
# Actually excluding directory symbolic links from |old_staging_links| is
# difficult because link targets might be already removed.
for path in set(list(old_staging_links) + list(new_staging_links)):
if path in old_staging_links and path in new_staging_links:
should_touch = old_staging_links[path] != new_staging_links[
path]
else:
should_touch = True
if should_touch and os.path.exists(path):
os.utime(path, None)
timer.done()
return True
def create_or_remove_bare_metal_gdb_lock_dir(gdb_target_list):
file_util.rmtree(_BARE_METAL_GDB_LOCK_DIR, ignore_errors=True)
if 'plugin' in gdb_target_list and OPTIONS.is_bare_metal_build():
file_util.makedirs_safely(_BARE_METAL_GDB_LOCK_DIR)
def _set_up_chromium_org_submodules():
CHROMIUM_ORG_ROOT = 'third_party/android/external/chromium_org'
# android/external/chromium_org contains these required submodules. It is not
# posible to have submodules within a submodule path (i.e., chromium_org)
# using git submodules. This is the list of subdirectories relative to
# chromium_org that we need to symlink to the appropriate submodules.
submodules = [
'sdch/open-vcdiff',
'testing/gtest',
'third_party/WebKit',
'third_party/angle',
'third_party/brotli/src',
('third_party/eyesfree/src/android/java/'
'src/com/googlecode/eyesfree/braille'),
'third_party/freetype',
'third_party/icu',
'third_party/leveldatabase/src',
'third_party/libjingle/source/talk',
'third_party/libphonenumber/src/phonenumbers',
'third_party/libphonenumber/src/resources',
'third_party/libsrtp',
'third_party/libvpx',
'third_party/libyuv',
'third_party/mesa/src',
'third_party/openmax_dl',
'third_party/openssl',
'third_party/opus/src',
'third_party/ots',
'third_party/sfntly/cpp/src',
'third_party/skia',
'third_party/smhasher/src',
'third_party/usrsctp/usrsctplib',
'third_party/webrtc',
'third_party/yasm/source/patched-yasm',
'v8']
# First remove all existing symlinks to make sure no stale links exist.
for dirpath, dirs, fnames in os.walk(CHROMIUM_ORG_ROOT):
# We only create symlinks for directories.
for name in dirs:
directory = os.path.join(dirpath, name)
if os.path.islink(directory):
os.unlink(directory)
for s in submodules:
target = os.path.join(CHROMIUM_ORG_ROOT, s)
# As an example, this maps 'sdch/open-vcdiff' to
# 'android/external/chromium_org__sdch_open-vcdiff', which is the true
# location of the submodule checkout.
source = 'third_party/android/external/chromium_org__' + s.replace('/', '_')
if not os.path.exists(source):
print 'ERROR: path "%s" does not exist.' % source
print 'ERROR: Did you forget to run git submodules update --init?'
sys.exit(1)
# Remove existing symlink for transition from previous version.
# The previous configuration script creates symlinks to the top of
# chromium.org submodules, and now it tries to symlink them one-level
# deeper.
#
# TODO(tzik): Remove this clean up code after the transition is no longer
# needed.
if os.path.islink(target):
os.remove(target)
target_contents = set()
try:
target_contents = set(os.listdir(target))
except OSError as e:
if e.errno != errno.ENOENT:
raise
source_contents = set(os.listdir(source))
for content in source_contents:
link_source = os.path.join(source, content)
link_target = os.path.join(target, content)
# If a real directory exists, remove it explicitly. |overwrite|
# flag does not care for real directories and files, but old symlinks.
if os.path.exists(link_target) and not os.path.islink(link_target):
file_util.rmtree(link_target)
file_util.create_link(link_target, link_source, overwrite=True)
for removed_item in target_contents.difference(source_contents):
os.unlink(os.path.join(target, removed_item))
def create_staging():
timer = build_common.SimpleTimer()
timer.start('Staging source files', True)
staging_root = build_common.get_staging_root()
# Store where all the old staging links pointed so we can compare after.
old_staging_links = _get_link_targets(staging_root)
if os.path.lexists(staging_root):
file_util.rmtree(staging_root)
_create_symlink_tree(_MODS_DIR, _THIRD_PARTY_DIR, staging_root)
# internal/ is an optional checkout
if build_options.OPTIONS.internal_apks_source_is_internal():
assert build_common.has_internal_checkout()
for name in os.listdir(_INTERNAL_THIRD_PARTY_PATH):
if os.path.exists(os.path.join(_THIRD_PARTY_DIR, name)):
raise Exception('Name conflict between internal/third_party and '
'third_party: ' + name)
_create_symlink_tree(_INTERNAL_MODS_PATH, _INTERNAL_THIRD_PARTY_PATH,
staging_root)
subprocess.check_call('internal/build/fix_staging.py')
# src/ is not overlaid on any directory.
_create_symlink_tree(_SRC_DIR, None, os.path.join(staging_root, 'src'))
# Update modification time for files that do not point to the same location
# that they pointed to in the previous tree to make sure they are built.
if old_staging_links:
new_staging_links = _get_link_targets(staging_root)
# Every file (not directory) under staging is in either one of following
# two states:
#
# F. The file itself is a symbolic link to a file under third_party or
# mods.
# D. Some ancestor directory is a symbolic link to a directory under
# third_party. (It is important that we do not create symbolic links to
# directories under mods)
#
# Let us say a file falls under "X-Y" case if it was in state X before
# re-staging and now in state Y. For all 4 possible cases, we can check if
# the actual destination of the file changed or not in the following way:
#
# F-F: We can just compare the target of the link.
# F-D, D-F: The target may have changed, but it needs some complicated
# computation to check. We treat them as changed to be conservative.
# D-D: We can leave it as-is since both point third_party.
#
# So we want to visit all files in state F either in old staging or new
# staging. For this purpose we can iterate through |*_staging_links| as
# they contain all files in state F.
#
# Note that |*_staging_links| may contain directory symbolic links, but
# it is okay to visit them too because directory timestamps do not matter.
# Actually excluding directory symbolic links from |old_staging_links| is
# difficult because link targets might be already removed.
for path in set(list(old_staging_links) + list(new_staging_links)):
if path in old_staging_links and path in new_staging_links:
should_touch = old_staging_links[path] != new_staging_links[path]
else:
should_touch = True
if should_touch and os.path.exists(path):
os.utime(path, None)
timer.done()
return True
def create_or_remove_bare_metal_gdb_lock_dir(gdb_target_list):
file_util.rmtree(_BARE_METAL_GDB_LOCK_DIR, ignore_errors=True)
if 'plugin' in gdb_target_list and OPTIONS.is_bare_metal_build():
file_util.makedirs_safely(_BARE_METAL_GDB_LOCK_DIR)
def setup_output_directory(output_dir):
"""Creates a directory to put all test log files."""
if os.path.exists(output_dir):
file_util.rmtree(output_dir)
file_util.makedirs_safely(output_dir)
def _download_adb(target, force):
"""Downloads the adb executable for Windows or Mac, if needed.
The downloaded executable will be placed at out/adb/win-x86_64/adb.exe or
out/adb/mac-x86_64/adb.
"""
# URL looks like 'https://dl.google.com/android/adt/adt-xxx.zip'
url = DEVTOOLS_URLS[target]
output_dir = os.path.join(ADB_OUTPUT_DIR, target)
stamp_file_path = os.path.join(output_dir, 'STAMP')
stamp_file = build_common.StampFile(
url, # Use URL as the version.
stamp_file_path,
force=force)
if stamp_file.is_up_to_date():
return
if os.path.exists(output_dir):
file_util.rmtree(output_dir)
os.makedirs(output_dir)
is_windows = target.startswith('win-')
adb_base_name = 'adb.exe' if is_windows else 'adb'
# The output file name looks like 'out/adb/win-x86_64/adb.exe'
adb_output_file_name = os.path.join(output_dir, adb_base_name)
zip_file_name = os.path.basename(urlparse.urlparse(url).path)
zip_name = os.path.splitext(zip_file_name)[0]
# The adb path in zip looks like 'adt-xxx/sdk/platform-tools/adb.exe'
adb_path_in_zip = os.path.join(zip_name, 'sdk/platform-tools',
adb_base_name)
# For Windows, AdbWinApi.dll is also needed.
if is_windows:
dll_path_in_zip = os.path.join(zip_name,
'sdk/platform-tools/AdbWinApi.dll')
dll_output_file_name = os.path.join(output_dir, 'AdbWinApi.dll')
try:
timer = build_common.SimpleTimer()
timer.start('Downloading the adb executable for %s' % target,
show=True)
with contextlib.closing(
urllib2.urlopen(url)) as stream, (zipfile.ZipFile(
cStringIO.StringIO(stream.read()))) as zip_archive:
with open(adb_output_file_name, 'w') as adb_file:
# Don't use zipfile.extract() as it creates sub directories.
content = zip_archive.read(adb_path_in_zip)
adb_file.write(content)
os.chmod(adb_output_file_name, stat.S_IRWXU)
# Also extract AdbWinApi.dll for Windows.
if is_windows:
with open(dll_output_file_name, 'w') as dll_file:
content = zip_archive.read(dll_path_in_zip)
dll_file.write(content)
os.chmod(dll_output_file_name, stat.S_IRWXU)
timer.done()
except Exception as exception:
print exception
raise Exception('Failed to download the adb executable')
stamp_file.update()