def UploadDexes(adb, execroot, app_dir, temp_dir, dexmanifest, full_install):
"""Uploads dexes to the device so that the state.
Does the minimum amount of work necessary to make the state of the device
consistent with what was built.
Args:
adb: the Adb instance representing the device to install to
execroot: the execroot
app_dir: the directory things should be installed under on the device
temp_dir: a local temporary directory
dexmanifest: contents of the dex manifest
full_install: whether to do a full install
Returns:
None.
"""
# Fetch the manifest on the device
dex_dir = os.path.join(app_dir, "dex")
adb.Mkdir(dex_dir)
old_manifest = None
if not full_install:
logging.info("Fetching dex manifest from device...")
old_manifest_contents = adb.Pull("%s/manifest" % dex_dir)
if old_manifest_contents:
old_manifest = ParseManifest(old_manifest_contents)
else:
logging.info("Dex manifest not found on device")
if old_manifest is None:
# If the manifest is not found, maybe a previous installation attempt
# was interrupted. Wipe the slate clean. Do this also in case we do a full
# installation.
old_manifest = {}
adb.Delete("%s/*" % dex_dir)
new_manifest = ParseManifest(dexmanifest)
dexes_to_delete = set(old_manifest) - set(new_manifest)
# Figure out which dexes to upload: those that are present in the new manifest
# but not in the old one and those whose checksum was changed
common_dexes = set(new_manifest).intersection(old_manifest)
dexes_to_upload = set(d for d in common_dexes
if new_manifest[d].sha256 != old_manifest[d].sha256)
dexes_to_upload.update(set(new_manifest) - set(old_manifest))
if not dexes_to_delete and not dexes_to_upload:
# If we have nothing to do, don't bother removing and rewriting the manifest
logging.info("Application dexes up-to-date")
return
# Delete the manifest so that we know how to get back to a consistent state
# if we are interrupted.
adb.Delete("%s/manifest" % dex_dir)
# Tuple of (local, remote) files to push to the device.
files_to_push = []
# Sort dexes to be uploaded by the zip file they are in so that we only need
# to open each zip only once.
dexzips_in_upload = set(new_manifest[d].input_file for d in dexes_to_upload
if new_manifest[d].zippath != "-")
for i, dexzip_name in enumerate(dexzips_in_upload):
zip_dexes = [
d for d in dexes_to_upload
if new_manifest[d].input_file == dexzip_name
]
dexzip_tempdir = os.path.join(temp_dir, "dex", str(i))
with zipfile.ZipFile(os.path.join(execroot, dexzip_name)) as dexzip:
for dex in zip_dexes:
zippath = new_manifest[dex].zippath
dexzip.extract(zippath, dexzip_tempdir)
files_to_push.append(
(os.path.join(dexzip_tempdir,
zippath), "%s/%s" % (dex_dir, dex)))
# Now gather all the dexes that are not within a .zip file.
dexes_to_upload = set(d for d in dexes_to_upload
if new_manifest[d].zippath == "-")
for dex in dexes_to_upload:
files_to_push.append(
(new_manifest[dex].input_file, "%s/%s" % (dex_dir, dex)))
num_files = len(dexes_to_delete) + len(files_to_push)
logging.info("Updating %d dex%s...", num_files,
"es" if num_files > 1 else "")
# Delete the dexes that are not in the new manifest
adb.DeleteMultiple(os.path.join(dex_dir, dex) for dex in dexes_to_delete)
# Upload all the files.
upload_walltime_start = time.time()
fs = [adb.Push(local, remote) for local, remote in files_to_push]
done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION)
upload_walltime = time.time() - upload_walltime_start
logging.debug("Dex upload walltime: %s seconds", upload_walltime)
# If there is anything in not_done, then some adb call failed and we
# can cancel the rest.
if not_done:
for f in not_done:
f.cancel()
# If any adb call resulted in an exception, re-raise it.
for f in done:
f.result()
# If no dex upload failed, upload the manifest. If any upload failed, the
# exception should have been re-raised above.
# Call result() to raise the exception if there was one.
adb.PushString(dexmanifest, "%s/manifest" % dex_dir).result()
def UploadNativeLibs(adb, native_lib_args, app_dir, full_install):
"""Uploads native libraries to the device."""
native_libs = ConvertNativeLibs(native_lib_args)
libs = set()
if native_libs:
abi = FindAbi(adb.GetAbi(), native_libs.keys())
if abi:
libs = native_libs[abi]
basename_to_path = {}
install_checksums = {}
for lib in sorted(libs):
install_checksums[os.path.basename(lib)] = Checksum(lib)
basename_to_path[os.path.basename(lib)] = lib
device_manifest = None
if not full_install:
device_manifest = adb.Pull("%s/native/native_manifest" % app_dir)
device_checksums = {}
if device_manifest is None:
# If we couldn't fetch the device manifest or if this is a non-incremental
# install, wipe the slate clean
adb.Delete("%s/native" % app_dir)
else:
# Otherwise, parse the manifest. Note that this branch is also taken if the
# manifest is empty.
for manifest_line in device_manifest.split("\n"):
if manifest_line:
name, checksum = manifest_line.split(" ")
device_checksums[name] = checksum
libs_to_delete = set(device_checksums) - set(install_checksums)
libs_to_upload = set(install_checksums) - set(device_checksums)
common_libs = set(install_checksums).intersection(set(device_checksums))
libs_to_upload.update([l for l in common_libs
if install_checksums[l] != device_checksums[l]])
libs_to_push = [(basename_to_path[lib], "%s/native/%s" % (app_dir, lib))
for lib in libs_to_upload]
if not libs_to_delete and not libs_to_push and device_manifest is not None:
logging.info("Native libs up-to-date")
return
num_files = len(libs_to_delete) + len(libs_to_push)
logging.info("Updating %d native lib%s...",
num_files, "s" if num_files != 1 else "")
adb.Delete("%s/native/native_manifest" % app_dir)
if libs_to_delete:
adb.DeleteMultiple([
"%s/native/%s" % (app_dir, lib) for lib in libs_to_delete])
upload_walltime_start = time.time()
fs = [adb.Push(local, remote) for local, remote in libs_to_push]
done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION)
upload_walltime = time.time() - upload_walltime_start
logging.debug("Native library upload walltime: %s seconds", upload_walltime)
# If there is anything in not_done, then some adb call failed and we
# can cancel the rest.
if not_done:
for f in not_done:
f.cancel()
# If any adb call resulted in an exception, re-raise it.
for f in done:
f.result()
install_manifest = [
name + " " + checksum for name, checksum in install_checksums.iteritems()]
adb.PushString("\n".join(install_manifest),
"%s/native/native_manifest" % app_dir).result()
def UploadDexes(adb, execroot, app_dir, temp_dir, dexmanifest, full_install):
"""Uploads dexes to the device so that the state.
Does the minimum amount of work necessary to make the state of the device
consistent with what was built.
Args:
adb: the Adb instance representing the device to install to
execroot: the execroot
app_dir: the directory things should be installed under on the device
temp_dir: a local temporary directory
dexmanifest: contents of the dex manifest
full_install: whether to do a full install
Returns:
None.
"""
# Fetch the manifest on the device
dex_dir = os.path.join(app_dir, "dex")
adb.Mkdir(dex_dir)
old_manifest = None
if not full_install:
logging.info("Fetching dex manifest from device...")
old_manifest_contents = adb.Pull("%s/manifest" % dex_dir)
if old_manifest_contents:
old_manifest = ParseManifest(old_manifest_contents)
else:
logging.info("Dex manifest not found on device")
if old_manifest is None:
# If the manifest is not found, maybe a previous installation attempt
# was interrupted. Wipe the slate clean. Do this also in case we do a full
# installation.
old_manifest = {}
adb.Delete("%s/*" % dex_dir)
new_manifest = ParseManifest(dexmanifest)
dexes_to_delete = set(old_manifest) - set(new_manifest)
# Figure out which dexes to upload: those that are present in the new manifest
# but not in the old one and those whose checksum was changed
common_dexes = set(new_manifest).intersection(old_manifest)
dexes_to_upload = set(d for d in common_dexes
if new_manifest[d].sha256 != old_manifest[d].sha256)
dexes_to_upload.update(set(new_manifest) - set(old_manifest))
if not dexes_to_delete and not dexes_to_upload:
# If we have nothing to do, don't bother removing and rewriting the manifest
logging.info("Application dexes up-to-date")
return
# Delete the manifest so that we know how to get back to a consistent state
# if we are interrupted.
adb.Delete("%s/manifest" % dex_dir)
# Tuple of (local, remote) files to push to the device.
files_to_push = []
# Sort dexes to be uploaded by the zip file they are in so that we only need
# to open each zip only once.
dexzips_in_upload = set(new_manifest[d].input_file for d in dexes_to_upload
if new_manifest[d].zippath != "-")
for i, dexzip_name in enumerate(dexzips_in_upload):
zip_dexes = [
d for d in dexes_to_upload if new_manifest[d].input_file == dexzip_name]
dexzip_tempdir = os.path.join(temp_dir, "dex", str(i))
with zipfile.ZipFile(os.path.join(execroot, dexzip_name)) as dexzip:
for dex in zip_dexes:
zippath = new_manifest[dex].zippath
dexzip.extract(zippath, dexzip_tempdir)
files_to_push.append(
(os.path.join(dexzip_tempdir, zippath), "%s/%s" % (dex_dir, dex)))
# Now gather all the dexes that are not within a .zip file.
dexes_to_upload = set(
d for d in dexes_to_upload if new_manifest[d].zippath == "-")
for dex in dexes_to_upload:
files_to_push.append(
(new_manifest[dex].input_file, "%s/%s" % (dex_dir, dex)))
num_files = len(dexes_to_delete) + len(files_to_push)
logging.info("Updating %d dex%s...", num_files, "es" if num_files > 1 else "")
# Delete the dexes that are not in the new manifest
adb.DeleteMultiple(os.path.join(dex_dir, dex) for dex in dexes_to_delete)
# Upload all the files.
upload_walltime_start = time.time()
fs = [adb.Push(local, remote) for local, remote in files_to_push]
done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION)
upload_walltime = time.time() - upload_walltime_start
logging.debug("Dex upload walltime: %s seconds", upload_walltime)
# If there is anything in not_done, then some adb call failed and we
# can cancel the rest.
if not_done:
for f in not_done:
f.cancel()
# If any adb call resulted in an exception, re-raise it.
for f in done:
f.result()
# If no dex upload failed, upload the manifest. If any upload failed, the
# exception should have been re-raised above.
# Call result() to raise the exception if there was one.
adb.PushString(dexmanifest, "%s/manifest" % dex_dir).result()
def UploadNativeLibs(adb, native_lib_args, app_dir, full_install):
"""Uploads native libraries to the device."""
native_libs = ConvertNativeLibs(native_lib_args)
libs = set()
if native_libs:
abi = FindAbi(adb.GetAbi(), native_libs.keys())
if abi:
libs = native_libs[abi]
basename_to_path = {}
install_checksums = {}
for lib in sorted(libs):
install_checksums[os.path.basename(lib)] = Checksum(lib)
basename_to_path[os.path.basename(lib)] = lib
device_manifest = None
if not full_install:
device_manifest = adb.Pull(
targetpath.join(app_dir, "native", "native_manifest"))
device_checksums = {}
if device_manifest is None:
# If we couldn't fetch the device manifest or if this is a non-incremental
# install, wipe the slate clean
adb.Delete(targetpath.join(app_dir, "native"))
else:
# Otherwise, parse the manifest. Note that this branch is also taken if the
# manifest is empty.
for manifest_line in device_manifest.split("\n"):
if manifest_line:
name, checksum = manifest_line.split(" ")
device_checksums[name] = checksum
libs_to_delete = set(device_checksums) - set(install_checksums)
libs_to_upload = set(install_checksums) - set(device_checksums)
common_libs = set(install_checksums).intersection(set(device_checksums))
libs_to_upload.update([
l for l in common_libs if install_checksums[l] != device_checksums[l]
])
libs_to_push = [(basename_to_path[lib],
targetpath.join(app_dir, "native", lib))
for lib in libs_to_upload]
if not libs_to_delete and not libs_to_push and device_manifest is not None:
logging.info("Native libs up-to-date")
return
num_files = len(libs_to_delete) + len(libs_to_push)
logging.info("Updating %d native lib%s...", num_files,
"s" if num_files != 1 else "")
adb.Delete(targetpath.join(app_dir, "native", "native_manifest"))
if libs_to_delete:
adb.DeleteMultiple([
targetpath.join(app_dir, "native", lib) for lib in libs_to_delete
])
upload_walltime_start = time.time()
fs = [adb.Push(local, remote) for local, remote in libs_to_push]
done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION)
upload_walltime = time.time() - upload_walltime_start
logging.debug("Native library upload walltime: %s seconds",
upload_walltime)
# If there is anything in not_done, then some adb call failed and we
# can cancel the rest.
if not_done:
for f in not_done:
f.cancel()
# If any adb call resulted in an exception, re-raise it.
for f in done:
f.result()
install_manifest = [
name + " " + checksum for name, checksum in install_checksums.items()
]
adb.PushString("\n".join(install_manifest),
targetpath.join(app_dir, "native",
"native_manifest")).result()
def UploadNativeLibs(adb, native_lib_args, app_dir, full_install):
"""Uploads native libraries to the device."""
native_libs = ConvertNativeLibs(native_lib_args)
libs = set()
if native_libs:
abi = FindAbi(adb.GetAbi(), native_libs.keys())
if abi:
libs = native_libs[abi]
basename_to_path = {}
install_checksums = {}
for lib in sorted(libs):
install_checksums[os.path.basename(lib)] = Checksum(lib)
basename_to_path[os.path.basename(lib)] = lib
device_manifest = None
if not full_install:
device_manifest = adb.Pull(
targetpath.join(app_dir, "native", "native_manifest"))
device_checksums = {}
if device_manifest is None:
# If we couldn't fetch the device manifest or if this is a non-incremental
# install, wipe the slate clean
adb.Delete(targetpath.join(app_dir, "native"))
# From Android 28 onwards, `adb push` creates directories with insufficient
# permissions, resulting in errors when pushing files. `adb shell mkdir`
# works correctly however, so we create the directory here.
# See https://github.com/bazelbuild/examples/issues/77 for more information.
adb.Mkdir(targetpath.join(app_dir, "native"))
else:
# Otherwise, parse the manifest. Note that this branch is also taken if the
# manifest is empty.
for manifest_line in device_manifest.split("\n"):
if manifest_line:
name, checksum = manifest_line.split(" ")
device_checksums[name] = checksum
libs_to_delete = set(device_checksums) - set(install_checksums)
libs_to_upload = set(install_checksums) - set(device_checksums)
common_libs = set(install_checksums).intersection(set(device_checksums))
libs_to_upload.update([l for l in common_libs
if install_checksums[l] != device_checksums[l]])
libs_to_push = [(basename_to_path[lib], targetpath.join(
app_dir, "native", lib)) for lib in libs_to_upload]
if not libs_to_delete and not libs_to_push and device_manifest is not None:
logging.info("Native libs up-to-date")
return
num_files = len(libs_to_delete) + len(libs_to_push)
logging.info("Updating %d native lib%s...",
num_files, "s" if num_files != 1 else "")
adb.Delete(targetpath.join(app_dir, "native", "native_manifest"))
if libs_to_delete:
adb.DeleteMultiple(
[targetpath.join(app_dir, "native", lib) for lib in libs_to_delete])
upload_walltime_start = time.time()
fs = [adb.Push(local, remote) for local, remote in libs_to_push]
done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION)
upload_walltime = time.time() - upload_walltime_start
logging.debug("Native library upload walltime: %s seconds", upload_walltime)
# If there is anything in not_done, then some adb call failed and we
# can cancel the rest.
if not_done:
for f in not_done:
f.cancel()
# If any adb call resulted in an exception, re-raise it.
for f in done:
f.result()
install_manifest = [
name + " " + checksum for name, checksum in install_checksums.items()]
adb.PushString("\n".join(install_manifest),
targetpath.join(app_dir, "native",
"native_manifest")).result()
def UploadNativeLibs(adb, native_lib_args, app_dir):
"""Uploads native libraries to the device."""
native_libs = ConvertNativeLibs(native_lib_args)
libs = set()
if native_libs:
abi = adb.GetAbi()
if abi not in native_libs:
logging.warn("No native libs for device ABI '%s'. Available ABIs: %s",
abi, ", ".join(native_libs))
else:
libs = native_libs[abi]
basename_to_path = {}
install_checksums = {}
for lib in sorted(libs):
install_checksums[os.path.basename(lib)] = Checksum(lib)
basename_to_path[os.path.basename(lib)] = lib
device_manifest = adb.Pull("%s/native/native_manifest" % app_dir)
device_checksums = {}
if device_manifest:
for name, checksum in [l.split(" ") for l in device_manifest.split("\n")]:
device_checksums[name] = checksum
libs_to_delete = set(device_checksums) - set(install_checksums)
libs_to_upload = set(install_checksums) - set(device_checksums)
common_libs = set(install_checksums).intersection(set(device_checksums))
libs_to_upload.update([l for l in common_libs
if install_checksums[l] != device_checksums[l]])
libs_to_push = [(basename_to_path[lib], "%s/native/%s" % (app_dir, lib))
for lib in libs_to_upload]
if not libs_to_delete and not libs_to_push and device_manifest is not None:
logging.info("Native libs up-to-date")
return
num_files = len(libs_to_delete) + len(libs_to_push)
logging.info("Updating %d native lib%s...",
num_files, "s" if num_files != 1 else "")
adb.Delete("%s/native/native_manifest" % app_dir)
if libs_to_delete:
adb.DeleteMultiple([
"%s/native/%s" % (app_dir, lib) for lib in libs_to_delete])
upload_walltime_start = time.time()
fs = [adb.Push(local, remote) for local, remote in libs_to_push]
done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION)
upload_walltime = time.time() - upload_walltime_start
logging.debug("Native library upload walltime: %s seconds", upload_walltime)
# If there is anything in not_done, then some adb call failed and we
# can cancel the rest.
if not_done:
for f in not_done:
f.cancel()
# If any adb call resulted in an exception, re-raise it.
for f in done:
f.result()
install_manifest = [
name + " " + checksum for name, checksum in install_checksums.iteritems()]
adb.PushString("\n".join(install_manifest),
"%s/native/native_manifest" % app_dir).result()
