def UploadDexes(adb, execroot, app_dir, temp_dir, dexmanifest, full_install): """Uploads dexes to the device so that the state. Does the minimum amount of work necessary to make the state of the device consistent with what was built. Args: adb: the Adb instance representing the device to install to execroot: the execroot app_dir: the directory things should be installed under on the device temp_dir: a local temporary directory dexmanifest: contents of the dex manifest full_install: whether to do a full install Returns: None. """ # Fetch the manifest on the device dex_dir = os.path.join(app_dir, "dex") adb.Mkdir(dex_dir) old_manifest = None if not full_install: logging.info("Fetching dex manifest from device...") old_manifest_contents = adb.Pull("%s/manifest" % dex_dir) if old_manifest_contents: old_manifest = ParseManifest(old_manifest_contents) else: logging.info("Dex manifest not found on device") if old_manifest is None: # If the manifest is not found, maybe a previous installation attempt # was interrupted. Wipe the slate clean. Do this also in case we do a full # installation. old_manifest = {} adb.Delete("%s/*" % dex_dir) new_manifest = ParseManifest(dexmanifest) dexes_to_delete = set(old_manifest) - set(new_manifest) # Figure out which dexes to upload: those that are present in the new manifest # but not in the old one and those whose checksum was changed common_dexes = set(new_manifest).intersection(old_manifest) dexes_to_upload = set(d for d in common_dexes if new_manifest[d].sha256 != old_manifest[d].sha256) dexes_to_upload.update(set(new_manifest) - set(old_manifest)) if not dexes_to_delete and not dexes_to_upload: # If we have nothing to do, don't bother removing and rewriting the manifest logging.info("Application dexes up-to-date") return # Delete the manifest so that we know how to get back to a consistent state # if we are interrupted. adb.Delete("%s/manifest" % dex_dir) # Tuple of (local, remote) files to push to the device. files_to_push = [] # Sort dexes to be uploaded by the zip file they are in so that we only need # to open each zip only once. dexzips_in_upload = set(new_manifest[d].input_file for d in dexes_to_upload if new_manifest[d].zippath != "-") for i, dexzip_name in enumerate(dexzips_in_upload): zip_dexes = [ d for d in dexes_to_upload if new_manifest[d].input_file == dexzip_name ] dexzip_tempdir = os.path.join(temp_dir, "dex", str(i)) with zipfile.ZipFile(os.path.join(execroot, dexzip_name)) as dexzip: for dex in zip_dexes: zippath = new_manifest[dex].zippath dexzip.extract(zippath, dexzip_tempdir) files_to_push.append( (os.path.join(dexzip_tempdir, zippath), "%s/%s" % (dex_dir, dex))) # Now gather all the dexes that are not within a .zip file. dexes_to_upload = set(d for d in dexes_to_upload if new_manifest[d].zippath == "-") for dex in dexes_to_upload: files_to_push.append( (new_manifest[dex].input_file, "%s/%s" % (dex_dir, dex))) num_files = len(dexes_to_delete) + len(files_to_push) logging.info("Updating %d dex%s...", num_files, "es" if num_files > 1 else "") # Delete the dexes that are not in the new manifest adb.DeleteMultiple(os.path.join(dex_dir, dex) for dex in dexes_to_delete) # Upload all the files. upload_walltime_start = time.time() fs = [adb.Push(local, remote) for local, remote in files_to_push] done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) upload_walltime = time.time() - upload_walltime_start logging.debug("Dex upload walltime: %s seconds", upload_walltime) # If there is anything in not_done, then some adb call failed and we # can cancel the rest. if not_done: for f in not_done: f.cancel() # If any adb call resulted in an exception, re-raise it. for f in done: f.result() # If no dex upload failed, upload the manifest. If any upload failed, the # exception should have been re-raised above. # Call result() to raise the exception if there was one. adb.PushString(dexmanifest, "%s/manifest" % dex_dir).result()
def UploadNativeLibs(adb, native_lib_args, app_dir, full_install): """Uploads native libraries to the device.""" native_libs = ConvertNativeLibs(native_lib_args) libs = set() if native_libs: abi = FindAbi(adb.GetAbi(), native_libs.keys()) if abi: libs = native_libs[abi] basename_to_path = {} install_checksums = {} for lib in sorted(libs): install_checksums[os.path.basename(lib)] = Checksum(lib) basename_to_path[os.path.basename(lib)] = lib device_manifest = None if not full_install: device_manifest = adb.Pull("%s/native/native_manifest" % app_dir) device_checksums = {} if device_manifest is None: # If we couldn't fetch the device manifest or if this is a non-incremental # install, wipe the slate clean adb.Delete("%s/native" % app_dir) else: # Otherwise, parse the manifest. Note that this branch is also taken if the # manifest is empty. for manifest_line in device_manifest.split("\n"): if manifest_line: name, checksum = manifest_line.split(" ") device_checksums[name] = checksum libs_to_delete = set(device_checksums) - set(install_checksums) libs_to_upload = set(install_checksums) - set(device_checksums) common_libs = set(install_checksums).intersection(set(device_checksums)) libs_to_upload.update([l for l in common_libs if install_checksums[l] != device_checksums[l]]) libs_to_push = [(basename_to_path[lib], "%s/native/%s" % (app_dir, lib)) for lib in libs_to_upload] if not libs_to_delete and not libs_to_push and device_manifest is not None: logging.info("Native libs up-to-date") return num_files = len(libs_to_delete) + len(libs_to_push) logging.info("Updating %d native lib%s...", num_files, "s" if num_files != 1 else "") adb.Delete("%s/native/native_manifest" % app_dir) if libs_to_delete: adb.DeleteMultiple([ "%s/native/%s" % (app_dir, lib) for lib in libs_to_delete]) upload_walltime_start = time.time() fs = [adb.Push(local, remote) for local, remote in libs_to_push] done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) upload_walltime = time.time() - upload_walltime_start logging.debug("Native library upload walltime: %s seconds", upload_walltime) # If there is anything in not_done, then some adb call failed and we # can cancel the rest. if not_done: for f in not_done: f.cancel() # If any adb call resulted in an exception, re-raise it. for f in done: f.result() install_manifest = [ name + " " + checksum for name, checksum in install_checksums.iteritems()] adb.PushString("\n".join(install_manifest), "%s/native/native_manifest" % app_dir).result()
def UploadDexes(adb, execroot, app_dir, temp_dir, dexmanifest, full_install): """Uploads dexes to the device so that the state. Does the minimum amount of work necessary to make the state of the device consistent with what was built. Args: adb: the Adb instance representing the device to install to execroot: the execroot app_dir: the directory things should be installed under on the device temp_dir: a local temporary directory dexmanifest: contents of the dex manifest full_install: whether to do a full install Returns: None. """ # Fetch the manifest on the device dex_dir = os.path.join(app_dir, "dex") adb.Mkdir(dex_dir) old_manifest = None if not full_install: logging.info("Fetching dex manifest from device...") old_manifest_contents = adb.Pull("%s/manifest" % dex_dir) if old_manifest_contents: old_manifest = ParseManifest(old_manifest_contents) else: logging.info("Dex manifest not found on device") if old_manifest is None: # If the manifest is not found, maybe a previous installation attempt # was interrupted. Wipe the slate clean. Do this also in case we do a full # installation. old_manifest = {} adb.Delete("%s/*" % dex_dir) new_manifest = ParseManifest(dexmanifest) dexes_to_delete = set(old_manifest) - set(new_manifest) # Figure out which dexes to upload: those that are present in the new manifest # but not in the old one and those whose checksum was changed common_dexes = set(new_manifest).intersection(old_manifest) dexes_to_upload = set(d for d in common_dexes if new_manifest[d].sha256 != old_manifest[d].sha256) dexes_to_upload.update(set(new_manifest) - set(old_manifest)) if not dexes_to_delete and not dexes_to_upload: # If we have nothing to do, don't bother removing and rewriting the manifest logging.info("Application dexes up-to-date") return # Delete the manifest so that we know how to get back to a consistent state # if we are interrupted. adb.Delete("%s/manifest" % dex_dir) # Tuple of (local, remote) files to push to the device. files_to_push = [] # Sort dexes to be uploaded by the zip file they are in so that we only need # to open each zip only once. dexzips_in_upload = set(new_manifest[d].input_file for d in dexes_to_upload if new_manifest[d].zippath != "-") for i, dexzip_name in enumerate(dexzips_in_upload): zip_dexes = [ d for d in dexes_to_upload if new_manifest[d].input_file == dexzip_name] dexzip_tempdir = os.path.join(temp_dir, "dex", str(i)) with zipfile.ZipFile(os.path.join(execroot, dexzip_name)) as dexzip: for dex in zip_dexes: zippath = new_manifest[dex].zippath dexzip.extract(zippath, dexzip_tempdir) files_to_push.append( (os.path.join(dexzip_tempdir, zippath), "%s/%s" % (dex_dir, dex))) # Now gather all the dexes that are not within a .zip file. dexes_to_upload = set( d for d in dexes_to_upload if new_manifest[d].zippath == "-") for dex in dexes_to_upload: files_to_push.append( (new_manifest[dex].input_file, "%s/%s" % (dex_dir, dex))) num_files = len(dexes_to_delete) + len(files_to_push) logging.info("Updating %d dex%s...", num_files, "es" if num_files > 1 else "") # Delete the dexes that are not in the new manifest adb.DeleteMultiple(os.path.join(dex_dir, dex) for dex in dexes_to_delete) # Upload all the files. upload_walltime_start = time.time() fs = [adb.Push(local, remote) for local, remote in files_to_push] done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) upload_walltime = time.time() - upload_walltime_start logging.debug("Dex upload walltime: %s seconds", upload_walltime) # If there is anything in not_done, then some adb call failed and we # can cancel the rest. if not_done: for f in not_done: f.cancel() # If any adb call resulted in an exception, re-raise it. for f in done: f.result() # If no dex upload failed, upload the manifest. If any upload failed, the # exception should have been re-raised above. # Call result() to raise the exception if there was one. adb.PushString(dexmanifest, "%s/manifest" % dex_dir).result()
def UploadNativeLibs(adb, native_lib_args, app_dir, full_install): """Uploads native libraries to the device.""" native_libs = ConvertNativeLibs(native_lib_args) libs = set() if native_libs: abi = FindAbi(adb.GetAbi(), native_libs.keys()) if abi: libs = native_libs[abi] basename_to_path = {} install_checksums = {} for lib in sorted(libs): install_checksums[os.path.basename(lib)] = Checksum(lib) basename_to_path[os.path.basename(lib)] = lib device_manifest = None if not full_install: device_manifest = adb.Pull( targetpath.join(app_dir, "native", "native_manifest")) device_checksums = {} if device_manifest is None: # If we couldn't fetch the device manifest or if this is a non-incremental # install, wipe the slate clean adb.Delete(targetpath.join(app_dir, "native")) else: # Otherwise, parse the manifest. Note that this branch is also taken if the # manifest is empty. for manifest_line in device_manifest.split("\n"): if manifest_line: name, checksum = manifest_line.split(" ") device_checksums[name] = checksum libs_to_delete = set(device_checksums) - set(install_checksums) libs_to_upload = set(install_checksums) - set(device_checksums) common_libs = set(install_checksums).intersection(set(device_checksums)) libs_to_upload.update([ l for l in common_libs if install_checksums[l] != device_checksums[l] ]) libs_to_push = [(basename_to_path[lib], targetpath.join(app_dir, "native", lib)) for lib in libs_to_upload] if not libs_to_delete and not libs_to_push and device_manifest is not None: logging.info("Native libs up-to-date") return num_files = len(libs_to_delete) + len(libs_to_push) logging.info("Updating %d native lib%s...", num_files, "s" if num_files != 1 else "") adb.Delete(targetpath.join(app_dir, "native", "native_manifest")) if libs_to_delete: adb.DeleteMultiple([ targetpath.join(app_dir, "native", lib) for lib in libs_to_delete ]) upload_walltime_start = time.time() fs = [adb.Push(local, remote) for local, remote in libs_to_push] done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) upload_walltime = time.time() - upload_walltime_start logging.debug("Native library upload walltime: %s seconds", upload_walltime) # If there is anything in not_done, then some adb call failed and we # can cancel the rest. if not_done: for f in not_done: f.cancel() # If any adb call resulted in an exception, re-raise it. for f in done: f.result() install_manifest = [ name + " " + checksum for name, checksum in install_checksums.items() ] adb.PushString("\n".join(install_manifest), targetpath.join(app_dir, "native", "native_manifest")).result()
def UploadNativeLibs(adb, native_lib_args, app_dir, full_install): """Uploads native libraries to the device.""" native_libs = ConvertNativeLibs(native_lib_args) libs = set() if native_libs: abi = FindAbi(adb.GetAbi(), native_libs.keys()) if abi: libs = native_libs[abi] basename_to_path = {} install_checksums = {} for lib in sorted(libs): install_checksums[os.path.basename(lib)] = Checksum(lib) basename_to_path[os.path.basename(lib)] = lib device_manifest = None if not full_install: device_manifest = adb.Pull( targetpath.join(app_dir, "native", "native_manifest")) device_checksums = {} if device_manifest is None: # If we couldn't fetch the device manifest or if this is a non-incremental # install, wipe the slate clean adb.Delete(targetpath.join(app_dir, "native")) # From Android 28 onwards, `adb push` creates directories with insufficient # permissions, resulting in errors when pushing files. `adb shell mkdir` # works correctly however, so we create the directory here. # See https://github.com/bazelbuild/examples/issues/77 for more information. adb.Mkdir(targetpath.join(app_dir, "native")) else: # Otherwise, parse the manifest. Note that this branch is also taken if the # manifest is empty. for manifest_line in device_manifest.split("\n"): if manifest_line: name, checksum = manifest_line.split(" ") device_checksums[name] = checksum libs_to_delete = set(device_checksums) - set(install_checksums) libs_to_upload = set(install_checksums) - set(device_checksums) common_libs = set(install_checksums).intersection(set(device_checksums)) libs_to_upload.update([l for l in common_libs if install_checksums[l] != device_checksums[l]]) libs_to_push = [(basename_to_path[lib], targetpath.join( app_dir, "native", lib)) for lib in libs_to_upload] if not libs_to_delete and not libs_to_push and device_manifest is not None: logging.info("Native libs up-to-date") return num_files = len(libs_to_delete) + len(libs_to_push) logging.info("Updating %d native lib%s...", num_files, "s" if num_files != 1 else "") adb.Delete(targetpath.join(app_dir, "native", "native_manifest")) if libs_to_delete: adb.DeleteMultiple( [targetpath.join(app_dir, "native", lib) for lib in libs_to_delete]) upload_walltime_start = time.time() fs = [adb.Push(local, remote) for local, remote in libs_to_push] done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) upload_walltime = time.time() - upload_walltime_start logging.debug("Native library upload walltime: %s seconds", upload_walltime) # If there is anything in not_done, then some adb call failed and we # can cancel the rest. if not_done: for f in not_done: f.cancel() # If any adb call resulted in an exception, re-raise it. for f in done: f.result() install_manifest = [ name + " " + checksum for name, checksum in install_checksums.items()] adb.PushString("\n".join(install_manifest), targetpath.join(app_dir, "native", "native_manifest")).result()
def UploadNativeLibs(adb, native_lib_args, app_dir): """Uploads native libraries to the device.""" native_libs = ConvertNativeLibs(native_lib_args) libs = set() if native_libs: abi = adb.GetAbi() if abi not in native_libs: logging.warn("No native libs for device ABI '%s'. Available ABIs: %s", abi, ", ".join(native_libs)) else: libs = native_libs[abi] basename_to_path = {} install_checksums = {} for lib in sorted(libs): install_checksums[os.path.basename(lib)] = Checksum(lib) basename_to_path[os.path.basename(lib)] = lib device_manifest = adb.Pull("%s/native/native_manifest" % app_dir) device_checksums = {} if device_manifest: for name, checksum in [l.split(" ") for l in device_manifest.split("\n")]: device_checksums[name] = checksum libs_to_delete = set(device_checksums) - set(install_checksums) libs_to_upload = set(install_checksums) - set(device_checksums) common_libs = set(install_checksums).intersection(set(device_checksums)) libs_to_upload.update([l for l in common_libs if install_checksums[l] != device_checksums[l]]) libs_to_push = [(basename_to_path[lib], "%s/native/%s" % (app_dir, lib)) for lib in libs_to_upload] if not libs_to_delete and not libs_to_push and device_manifest is not None: logging.info("Native libs up-to-date") return num_files = len(libs_to_delete) + len(libs_to_push) logging.info("Updating %d native lib%s...", num_files, "s" if num_files != 1 else "") adb.Delete("%s/native/native_manifest" % app_dir) if libs_to_delete: adb.DeleteMultiple([ "%s/native/%s" % (app_dir, lib) for lib in libs_to_delete]) upload_walltime_start = time.time() fs = [adb.Push(local, remote) for local, remote in libs_to_push] done, not_done = futures.wait(fs, return_when=futures.FIRST_EXCEPTION) upload_walltime = time.time() - upload_walltime_start logging.debug("Native library upload walltime: %s seconds", upload_walltime) # If there is anything in not_done, then some adb call failed and we # can cancel the rest. if not_done: for f in not_done: f.cancel() # If any adb call resulted in an exception, re-raise it. for f in done: f.result() install_manifest = [ name + " " + checksum for name, checksum in install_checksums.iteritems()] adb.PushString("\n".join(install_manifest), "%s/native/native_manifest" % app_dir).result()