def delete_library(name, force=False):
"""Remove a library"""
rtn = 0
# First check if we know about the library
if is_library_installed(name):
item = dtf.core.item.Item()
item.name = name
item.type = dtf.core.item.TYPE_LIBRARY
installed_item = __load_item(item)
# Prompt for removal
if not force:
if __prompt_delete(installed_item):
log.d(TAG, "User would like to remove")
rtn = __do_library_delete(installed_item)
else:
log.i(TAG, "Library deletion skipped.")
# Force
else:
log.d(TAG, "Forcing component removal.")
rtn = __do_library_delete(installed_item)
else:
log.e(TAG, "No library installed with this name.")
rtn = -1
return rtn
def delete_module(name, force=False):
"""Remove a module"""
rtn = 0
# First check if we know about the module
if is_module_installed(name):
item = Item()
item.name = name
item.type = TYPE_MODULE
installed_item = __load_item(item)
# Prompt for removal
if not force:
if __prompt_delete(installed_item):
log.d(TAG, "User would like to remove")
rtn = __do_module_delete(installed_item)
else:
log.i(TAG, "Module deletion skipped.")
# Force
else:
log.d(TAG, "Forcing component removal.")
rtn = __do_module_delete(installed_item)
else:
log.e(TAG, "No module installed with this name.")
rtn = -1
return rtn
def copy_tree(local_name, install_name, install_dir):
"""Copy a directory recursively"""
install_path = install_dir + install_name + '/'
# We need to remove the first one
rmtree(install_path, ignore_errors=True)
# Make the new directory.
os.makedirs(install_path)
for root, dirs, files in os.walk(local_name):
# Create new directories.
if len(dirs) != 0:
for local_dir in [os.path.join(root, name) for name in dirs]:
new_dir = local_dir.replace(local_name + '/', '', 1)
log.d(TAG, "Making dir %s" % (install_path + new_dir))
os.makedirs(install_path + new_dir)
if len(files) != 0:
for local_file in [os.path.join(root, name) for name in files]:
new_file = local_file.replace(local_name + '/', '', 1)
log.d(
TAG, "Copying file '%s' to '%s'" %
(local_file, install_path + new_file))
copy(local_file, install_path + new_file)
return 0
def delete_package(name, force=False):
"""Remove a package"""
rtn = 0
# First check if we know about the package
if is_package_installed(name):
item = dtf.core.item.Item()
item.name = name
item.type = dtf.core.item.TYPE_PACKAGE
installed_item = __load_item(item)
# Prompt for removal
if not force:
if __prompt_delete(installed_item):
log.d(TAG, "User would like to remove")
rtn = __do_package_delete(installed_item)
else:
log.i(TAG, "Package deletion skipped.")
# Force
else:
log.d(TAG, "Forcing component removal.")
rtn = __do_package_delete(installed_item)
else:
log.e(TAG, "No package installed with this name.")
rtn = -1
return rtn
def delete_package(name, force=False):
"""Remove a package"""
rtn = 0
# First check if we know about the package
if is_package_installed(name):
item = Item()
item.name = name
item.type = TYPE_PACKAGE
installed_item = __load_item(item)
# Prompt for removal
if not force:
if __prompt_delete(installed_item):
log.d(TAG, "User would like to remove")
rtn = __do_package_delete(installed_item)
else:
log.i(TAG, "Package deletion skipped.")
# Force
else:
log.d(TAG, "Forcing component removal.")
rtn = __do_package_delete(installed_item)
else:
log.e(TAG, "No package installed with this name.")
rtn = -1
return rtn
def delete_library(name, force=False):
"""Remove a library"""
rtn = 0
# First check if we know about the library
if is_library_installed(name):
item = Item()
item.name = name
item.type = TYPE_LIBRARY
installed_item = __load_item(item)
# Prompt for removal
if not force:
if __prompt_delete(installed_item):
log.d(TAG, "User would like to remove")
rtn = __do_library_delete(installed_item)
else:
log.i(TAG, "Library deletion skipped.")
# Force
else:
log.d(TAG, "Forcing component removal.")
rtn = __do_library_delete(installed_item)
else:
log.e(TAG, "No library installed with this name.")
rtn = -1
return rtn
def do_python_checks(file_name, module_name, all_checks, strict_checks):
"""Run all python checks"""
# First attempt to auto parse
item = pm.parse_python_module(file_name, module_name)
if item is None:
log.e(TAG, "[FAIL] Auto parse failed!")
return -1
if do_auto_checks(item, strict_checks) != 0:
return -1
if all_checks:
log.d(TAG, "Running pylint...")
if run_command("pylint \"%s\"" % file_name) != 0:
log.e(TAG, "[FAIL] pylint failed.")
return -1
log.d(TAG, "Running flake8...")
if run_command("flake8 \"%s\"" % file_name) != 0:
log.e(TAG, "[FAIL] flake8 failed.")
return -1
log.i(TAG, "[PASS] All checks passed!")
return 0
def do_bash_checks(file_name, module_name, all_checks, strict_checks):
"""Run app bash checks"""
# First attempt to execute.
if not utils.is_executable(file_name):
log.e(TAG, "[FAIL] Module is not marked executable!")
return -1
# Next attempt to auto parse
item = pm.parse_bash_module(file_name, module_name)
if item is None:
log.e(TAG, "[FAIL] Auto parse failed!")
return -1
if do_auto_checks(item, strict_checks) != 0:
return -1
if all_checks:
log.d(TAG, "Running checkbashisms...")
if run_command("checkbashisms -f \"%s\"" % file_name) != 0:
log.e(TAG, "[FAIL] checkbashisms failed.")
return -1
log.d(TAG, "Running shellcheck...")
if run_command("shellcheck \"%s\"" % file_name) != 0:
log.e(TAG, "[FAIL] shellcheck failed.")
return -1
log.i(TAG, "[PASS] All checks passed!")
return 0
def copy_tree(local_name, install_name, install_dir):
"""Copy a directory recursively"""
install_path = install_dir + install_name + '/'
# We need to remove the first one
rmtree(install_path, ignore_errors=True)
# Make the new directory.
makedirs(install_path)
for root, dirs, files in os.walk(local_name):
# Create new directories.
if len(dirs) != 0:
for local_dir in [os.path.join(root, name) for name in dirs]:
new_dir = local_dir.replace(local_name+'/', '', 1)
log.d(TAG, "Making dir %s" % (install_path + new_dir))
makedirs(install_path + new_dir)
if len(files) != 0:
for local_file in [os.path.join(root, name) for name in files]:
new_file = local_file.replace(local_name+'/', '', 1)
log.d(TAG, "Copying file '%s' to '%s'"
% (local_file, install_path + new_file))
copy(local_file, install_path + new_file)
return 0
def test_debug():
"""Log a debug message"""
log.LOG_LEVEL_STDOUT = 5
log.LOG_LEVEL_FILE = 5
log.d(TAG, "This is an debug!")
def test_debug_suppress():
"""Log a debug message with logging at 4"""
log.LOG_LEVEL_STDOUT = 4
log.LOG_LEVEL_FILE = 4
log.d(TAG, "This is a debug message!")
def __add_file(self, subdir, item):
"""Add a file to the correct subdirectory"""
install_to = "%s/%s" % (subdir, item.install_name)
log.d(TAG, "Adding '%s' as '%s'" % (item.install_name, install_to))
self.zip_f.write(item.local_name, install_to)
def __do_upload(self, local_file_name, remote_file_name):
"""Do file upload"""
# Get a connected socket
sock = self.__sock_connect(DTF_SOCKET)
if sock is None:
log.e(TAG, "Cannot __do_upload, socket failure.")
return ERR_SOCK
statinfo = os.stat(local_file_name)
file_size = statinfo.st_size
local_f = open(local_file_name, 'rb')
sock.send(CMD_UPLOAD)
resp_code = self.__safe_recv(sock, 1, response=RESP_ERROR)
if resp_code != RESP_OK:
log.e(TAG, "Server rejected upload request!")
return resp_code
log.d(TAG, "Sending filesize to server")
sock.send(long_to_bytes(file_size))
resp = sock.recv(1)
if resp != RESP_OK:
log.e(TAG, "Error submitting filesize!")
return resp
padded_file_name = remote_file_name.ljust(SIZE_FILENAME, '\0')
log.d(TAG, "Sending the filename...")
sock.send(padded_file_name)
resp = self.__safe_recv(sock, 1, response=RESP_ERROR)
if resp != RESP_OK:
log.e(TAG, "Error with filename!")
return resp
bytes_left = file_size
while True:
if bytes_left <= SIZE_TRANSFER:
sock.send(local_f.read(bytes_left))
local_f.close()
break
else:
sock.send(local_f.read(SIZE_TRANSFER))
bytes_left -= SIZE_TRANSFER
resp = self.__safe_recv(sock, 1, response=RESP_ERROR)
if resp != RESP_OK:
log.e(TAG, "Error uploading file!")
return resp
return RESP_OK
def parse_single_item(self, args): # pylint: disable=too-many-branches
"""Parse args, return Item"""
item = dtf.core.item.Item()
if args.single_name is None:
log.e(TAG, "No '--name' specified in single item mode. Exiting.")
return None
item.name = args.single_name
if args.single_type not in dtf.core.item.VALID_TYPES:
log.e(TAG, "Invalid type passed to single. Exiting.")
return None
item.type = args.single_type
version = args.single_version
if version is not None:
if dtf.core.item.is_valid_version(version):
item.version = version
else:
log.e(TAG, "Version string is not valid. Exiting.")
return None
else:
log.w(TAG, "No version provided, using v1.0.0")
item.version = "1.0.0"
try:
item.author = " ".join(args.single_author)
except TypeError:
item.author = None
try:
item.about = " ".join(args.single_about)
except TypeError:
item.about = None
install_name = args.single_install_name
local_name = args.single_local_name
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = os.path.basename(args.single_name)
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = args.single_name
item.install_name = install_name
item.local_name = local_name
if self.check_local_exists(item):
return item
else:
return None
def __process_zip_items(zip_file, items, force):
"""Process items in a ZIP"""
for item in items:
item_type = get_xml_attrib(item, "type")
if item_type is None or item_type not in VALID_TYPES:
log.e(TAG, "Found tag with no 'type' attribute, skipping!")
continue
if item_type not in VALID_TYPES:
log.e(TAG, "Illegal 'type' attribute found, skipping!")
continue
name = get_xml_attrib(item, "name")
if name is None:
log.e(TAG, "Found NULL named moduled, skipping!")
continue
# Ok, lets start. We can generically parse.
local_item = Item()
local_item.type = item_type
local_item.major_version = get_xml_attrib(item, "majorVersion")
local_item.minor_version = get_xml_attrib(item, "minorVersion")
local_item.health = get_xml_attrib(item, "health")
local_item.author = get_xml_attrib(item, "author")
local_item.about = get_xml_attrib(item, "about")
install_name = get_xml_attrib(item, "installName")
local_name = get_xml_attrib(item, "localName")
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = name
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = name
local_item.name = name
local_item.install_name = install_name
local_item.local_name = local_name
if item_type == TYPE_BINARY:
rtn = __install_zip_binary(zip_file, local_item, force)
elif item_type == TYPE_LIBRARY:
rtn = __install_zip_library(zip_file, local_item, force)
elif item_type == TYPE_MODULE:
rtn = __install_zip_module(zip_file, local_item, force)
elif item_type == TYPE_PACKAGE:
rtn = __install_zip_package(zip_file, local_item, force)
return rtn
def __process_zip_items(zip_file, items, force):
"""Process items in a ZIP"""
for item in items:
item_type = get_xml_attrib(item, "type")
if item_type is None or item_type not in VALID_TYPES:
log.e(TAG, "Found tag with no 'type' attribute, skipping!")
continue
if item_type not in VALID_TYPES:
log.e(TAG, "Illegal 'type' attribute found, skipping!")
continue
name = get_xml_attrib(item, "name")
if name is None:
log.e(TAG, "Found NULL named moduled, skipping!")
continue
# Ok, lets start. We can generically parse.
local_item = Item()
local_item.type = item_type
local_item.major_version = get_xml_attrib(item, "majorVersion")
local_item.minor_version = get_xml_attrib(item, "minorVersion")
local_item.health = get_xml_attrib(item, "health")
local_item.author = get_xml_attrib(item, "author")
local_item.about = get_xml_attrib(item, "about")
install_name = get_xml_attrib(item, "installName")
local_name = get_xml_attrib(item, "localName")
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = name
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = name
local_item.name = name
local_item.install_name = install_name
local_item.local_name = local_name
if item_type == TYPE_BINARY:
rtn = __install_zip_binary(zip_file, local_item, force)
elif item_type == TYPE_LIBRARY:
rtn = __install_zip_library(zip_file, local_item, force)
elif item_type == TYPE_MODULE:
rtn = __install_zip_module(zip_file, local_item, force)
elif item_type == TYPE_PACKAGE:
rtn = __install_zip_package(zip_file, local_item, force)
return rtn
def launch_local_module(root, cmd, args):
"""Launch a local module"""
module_path = "%s/%s/%s" % (root, utils.LOCAL_MODULES_DIRECTORY, cmd)
# If we are dealing with a bash script, just run and exit.
if pm.is_bash_module(module_path):
log.d(TAG, "This is a bash module!")
return __launch_bash_module(module_path, args)
return __launch_python_module(module_path, cmd, args)
def launch_local_module(root, cmd, args):
"""Launch a local module"""
module_path = "%s/local_modules/%s" % (root, cmd)
# If we are dealing with a bash script, just run and exit.
if pm.is_bash_module(module_path):
log.d(TAG, "This is a bash module!")
return __launch_bash_module(module_path, args)
return __launch_python_module(module_path, cmd, args)
def copy_file(local_name, install_name, install_dir):
"""Copy a file"""
install_path = install_dir + install_name
log.d(TAG, "Copying '%s' to '%s'..." % (local_name, install_path))
copy(local_name, install_path)
os.chmod(install_path, 0o755)
log.d(TAG, "Copy complete!")
return 0
def __unpack_included(tar_path):
"""Unzip the included TAR"""
log.d(TAG, "Doing unpack from local...")
utils.mkdir_recursive(dtfglobals.DTF_INCLUDED_DIR)
tar_path = "%s/included.tar" % utils.get_dtf_lib_dir()
with tarfile.TarFile(tar_path, 'r') as included_tar:
included_tar.extractall(dtfglobals.DTF_INCLUDED_DIR)
return 0
def copy_file(local_name, install_name, install_dir):
"""Copy a file"""
install_path = install_dir + install_name
log.d(TAG, "Copying '%s' to '%s'..." % (local_name, install_path))
copy(local_name, install_path)
chmod(install_path, 0755)
log.d(TAG, "Copy complete!")
return 0
def __add_tree(self, subdir, item):
"""Add entire tree to ZIP"""
for root, _, files in os.walk(item.local_name):
for file_name in files:
file_path = os.path.join(root, file_name)
stripped_path = file_path.replace(item.local_name, "", 1)
install_to = os.path.normpath(
"%s/%s/%s" % (subdir, item.name, stripped_path))
log.d(TAG, "Adding dir '%s' as '%s'" % (file_path, install_to))
self.zip_f.write(file_path, install_to)
def __generic_install(item, force_mode, new_only, check_function,
install_function, install_args):
"""Generic check and caller"""
try:
# First check if we know about this item
if check_function(item.name):
log.d(TAG, "Item exists, need to check")
# If forced, don't even check.
if force_mode:
log.i(
TAG, "Forcing component installation: %s (%s)" %
(item.name, item.type))
return install_function(*install_args)
installed_item = __load_item(item)
# Ok, next, we check the versions.
if dtf.core.item.item_is_newer(installed_item, item):
log.i(
TAG, "Upgrading %s from v%s to v%s" %
(item.name, installed_item.version, item.version))
return install_function(*install_args)
elif new_only:
log.w(TAG, "Skipping due to older version: %s" % item.name)
return 0
# Otherwise we need to prompt
else:
print("[WARNING] An item with this name is already installed."
" See details below.")
if __prompt_install(item, installed_item):
log.d(TAG, "User would like to install")
return install_function(*install_args)
else:
log.w(TAG, "Installation skipped.")
return 0
else:
log.i(TAG, "Installing new item: %s (%s)" % (item.name, item.type))
return install_function(*install_args)
except KeyError:
log.w(TAG, "Error checking if the item was installed. Skipping")
return -4
def __item_from_xml(item, relative_root="./"):
"""create Item object from XML"""
item_type = __get_xml_attrib(item, "type")
if item_type is None:
log.e(TAG, "Found tag with no 'type' attribute, skipping!")
return None
if item_type not in dtf.core.item.VALID_TYPES:
log.e(TAG, "Illegal 'type' attribute found, skipping!")
return None
name = __get_xml_attrib(item, "name")
if name is None:
log.e(TAG, "Found NULL named moduled, skipping!")
return None
# Ok, lets start. We can generically parse.
local_item = dtf.core.item.Item()
local_item.type = item_type
local_item.version = __get_xml_attrib(item, "version")
local_item.author = __get_xml_attrib(item, "author")
local_item.about = __get_xml_attrib(item, "about")
if local_item.version is None:
log.w(TAG, "No version for '%s', using 1.0.0" % name)
local_item.version = "1.0.0"
install_name = __get_xml_attrib(item, "installName")
local_name = __get_xml_attrib(item, "localName")
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = name
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = name
else:
local_name = os.path.normpath("%s/%s" % (relative_root, local_name))
local_item.name = name
local_item.install_name = install_name
local_item.local_name = local_name
return local_item
def __copy_zip_tree(self, local_name, install_name, install_dir):
"""Copy directory in ZIP to directory"""
install_path = install_dir + install_name + '/'
# We need to remove the first one
rmtree(install_path, ignore_errors=True)
reduced_list = [
file_f for file_f in self.zip_f.namelist()
if file_f.startswith(local_name) and file_f != local_name + '/'
]
self.zip_f.extractall(dtf.globals.DTF_DATA_DIR, reduced_list)
log.d(TAG, "Copy complete!")
return 0
def determine_vm_type(self, sdk, cpu_bits):
"""Determine if we are Dalvik/ART"""
# ART was introduced in KitKat, so if we are less, its Dalvik.
if int(sdk) < 20:
log.d(TAG, "Using Dalvik based on SDK")
return TYPE_DALVIK
# Check for single persist.sys.dalvik.vm.lib
lib = self.getprop('persist.sys.dalvik.vm.lib')
lib2 = self.getprop('persist.sys.dalvik.vm.lib.2')
# None set, error
if lib == '' and lib2 == '':
log.e(TAG, "Unable to determine VM type!")
return None
# Both are set.
elif lib != '' and lib2 != '':
if cpu_bits == '64':
arm_dir = '/system/framework/arm64'
else:
arm_dir = '/system/framework/arm'
if self.adb.is_dir(arm_dir):
log.d(TAG, "Using ART based ARM directory.")
return TYPE_ART
else:
log.d(TAG, "Using Dalvik based on ARM directory.")
return TYPE_DALVIK
# One or the other is set.
else:
so_type = max([lib, lib2])
if so_type == 'libart.so':
log.d(TAG, "Using ART based on prop.")
return TYPE_ART
else:
log.d(TAG, "Using Dalvik based on prop.")
return TYPE_DALVIK
def determine_vm_type(self, sdk, cpu_bits):
"""Determine if we are Dalvik/ART"""
# ART was introduced in KitKat, so if we are less, its Dalvik.
if int(sdk) < 20:
log.d(TAG, "Using Dalvik based on SDK")
return TYPE_DALVIK
# Check for single persist.sys.dalvik.vm.lib
lib = self.getprop('persist.sys.dalvik.vm.lib')
lib2 = self.getprop('persist.sys.dalvik.vm.lib.2')
# None set, error
if lib == '' and lib2 == '':
log.e(TAG, "Unable to determine VM type!")
return None
# Both are set.
elif lib != '' and lib2 != '':
if cpu_bits == '64':
arm_dir = '/system/framework/arm64'
else:
arm_dir = '/system/framework/arm'
if self.adb.is_dir(arm_dir):
log.d(TAG, "Using ART based ARM directory.")
return TYPE_ART
else:
log.d(TAG, "Using Dalvik based on ARM directory.")
return TYPE_DALVIK
# One or the other is set.
else:
so_type = max([lib, lib2])
if so_type == 'libart.so':
log.d(TAG, "Using ART based on prop.")
return TYPE_ART
else:
log.d(TAG, "Using Dalvik based on prop.")
return TYPE_DALVIK
def install_single_package(item, force=False):
"""Install a single package"""
rtn = 0
try:
# First check if we know about this package
if is_binary_installed(item.name):
# Prompt for install.
if not force:
print(
"[WARNING] An item with this name is already installed."
" See details below.")
installed_item = __load_item(item)
if __prompt_install(item, installed_item):
log.d(TAG, "User would like to install")
rtn = __do_single_package_install(item)
else:
log.i(TAG, "Package installation skipped.")
# Force
else:
log.d(TAG, "Forcing component installation.")
rtn = __do_single_package_install(item)
else:
log.d(TAG, "This is a new package, installing.")
rtn = __do_single_package_install(item)
except KeyError:
log.w(TAG, "Error checking if the package was installed. Skipping")
rtn = -4
return rtn
def install_zip(zip_file_name, force=False):
"""Install a ZIP file"""
rtn = 0
manifest_data = None
manifest_root = None
zip_f = ZipFile(zip_file_name, 'r')
log.d(TAG, "Parsing manifest file...")
# Get Manifest
if not file_in_zip(zip_f, MANIFEST_NAME):
log.e(TAG, "Error extracting '%s' from ZIP archive - does it exist?"
% (MANIFEST_NAME))
return -3
manifest_data = zip_f.read(MANIFEST_NAME)
# Read Manifest
try:
manifest_root = etree.XML(manifest_data)
except etree.XMLSyntaxError as err:
log.e(TAG, "Error parsing XML file '%s'! Exiting."
% MANIFEST_NAME)
return -4
# Processing Stuff
items = manifest_root.xpath("/Items/Item")
rtn = __process_zip_items(zip_f, items, force)
if rtn == 0:
log.i(TAG, "ZIP content '%s' installed successfully!"
% zip_file_name)
else:
log.e(TAG, "Unable to install ZIP file: %d" % rtn)
return rtn
def copy_zip_tree(zip_f, local_name, install_name, install_dir):
"""Copy directory in ZIP to directory"""
install_path = install_dir + install_name + '/'
# We need to remove the first one
rmtree(install_path, ignore_errors=True)
# Make the new directory.
makedirs(install_path)
for file_f in zip_f.namelist():
# Do everything in the [local_name] dir, but not the root.
if file_f.startswith(local_name) and file_f != local_name + '/':
# First, we need to remove the first element.
new_f = file_f.replace(local_name + '/', '', 1)
# If it's a directory, make it.
if file_f.endswith('/'):
log.d(TAG, "Making dir %s" % install_path + new_f)
makedirs(install_path + new_f)
# Otherwise, we need to unzip to that new path.
else:
head, tail = os.path.split(new_f)
log.d(TAG,
"extracting %s to %s" % (file_f, install_path + head))
copy_zip_file(zip_f, file_f, tail, install_path + head + '/')
log.d(TAG, "Copy complete!")
return 0
def install_single_package(item, force=False):
"""Install a single package"""
rtn = 0
try:
# First check if we know about this package
if is_binary_installed(item.name):
# Prompt for install.
if not force:
print ("[WARNING] An item with this name is already installed."
" See details below.")
installed_item = __load_item(item)
if __prompt_install(item, installed_item):
log.d(TAG, "User would like to install")
rtn = __do_single_package_install(item)
else:
log.i(TAG, "Package installation skipped.")
# Force
else:
log.d(TAG, "Forcing component installation.")
rtn = __do_single_package_install(item)
else:
log.d(TAG, "This is a new package, installing.")
rtn = __do_single_package_install(item)
except KeyError:
log.w(TAG, "Error checking if the package was installed. Skipping")
rtn = -4
return rtn
def do_checks(file_name, all_checks, strict_checks):
"""Perform checks"""
# Update path to include libs
update_path()
if not os.path.isfile(file_name):
log.e(TAG, "[FAIL] %s is not a file." % file_name)
return -1
base_name = os.path.basename(file_name)
module_name = os.path.splitext(base_name)[0]
log.d(TAG, "Full File: %s" % file_name)
log.d(TAG, "Base name: %s" % base_name)
log.d(TAG, "Module name: %s" % module_name)
# First, is this python, or bash?
if pm.is_python_module(file_name, module_name):
log.i(TAG, "[PASS] Is python, doing python checks...")
return do_python_checks(file_name, module_name, all_checks,
strict_checks)
elif pm.is_bash_module(file_name):
log.i(TAG, "[PASS] Is bash, doing bash checks...")
return do_bash_checks(file_name, module_name, all_checks,
strict_checks)
else:
log.e(TAG, "[FAIL] This is not recognized as either python or bash!")
return -2
def copy_zip_tree(zip_f, local_name, install_name, install_dir):
"""Copy directory in ZIP to directory"""
install_path = install_dir + install_name + '/'
# We need to remove the first one
rmtree(install_path, ignore_errors=True)
# Make the new directory.
makedirs(install_path)
for file_f in zip_f.namelist():
# Do everything in the [local_name] dir, but not the root.
if file_f.startswith(local_name) and file_f != local_name+'/':
# First, we need to remove the first element.
new_f = file_f.replace(local_name+'/', '', 1)
# If it's a directory, make it.
if file_f.endswith('/'):
log.d(TAG, "Making dir %s" % install_path + new_f)
makedirs(install_path + new_f)
# Otherwise, we need to unzip to that new path.
else:
head, tail = os.path.split(new_f)
log.d(TAG, "extracting %s to %s"
%(file_f, install_path + head))
copy_zip_file(zip_f, file_f, tail, install_path + head + '/')
log.d(TAG, "Copy complete!")
return 0
def copy_zip_file(zip_f, local_name, install_name, install_dir):
"""Copy file in ZIP to directory"""
install_path = install_dir + install_name
log.d(TAG, "Copying '%s' to '%s'..." % (local_name, install_path))
temp_f = NamedTemporaryFile(mode='w', delete=False)
temp_f.write(zip_f.read(local_name))
temp_f.flush()
copy(temp_f.name, install_path)
chmod(install_path, 0755)
temp_f.close()
remove(temp_f.name)
log.d(TAG, "Copy complete!")
return 0
def install_zip(zip_file_name, force=False):
"""Install a ZIP file"""
rtn = 0
manifest_data = None
manifest_root = None
zip_f = ZipFile(zip_file_name, 'r')
log.d(TAG, "Parsing manifest file...")
# Get Manifest
if not file_in_zip(zip_f, MANIFEST_NAME):
log.e(
TAG, "Error extracting '%s' from ZIP archive - does it exist?" %
(MANIFEST_NAME))
return -3
manifest_data = zip_f.read(MANIFEST_NAME)
# Read Manifest
try:
manifest_root = etree.XML(manifest_data)
except etree.XMLSyntaxError as err:
log.e(TAG, "Error parsing XML file '%s'! Exiting." % MANIFEST_NAME)
return -4
# Processing Stuff
items = manifest_root.xpath("/Items/Item")
rtn = __process_zip_items(zip_f, items, force)
if rtn == 0:
log.i(TAG, "ZIP content '%s' installed successfully!" % zip_file_name)
else:
log.e(TAG, "Unable to install ZIP file: %d" % rtn)
return rtn
def copy_zip_file(zip_f, local_name, install_name, install_dir):
"""Copy file in ZIP to directory"""
install_path = install_dir + install_name
log.d(TAG, "Copying '%s' to '%s'..." % (local_name, install_path))
temp_f = NamedTemporaryFile(mode='w', delete=False)
temp_f.write(zip_f.read(local_name))
temp_f.flush()
copy(temp_f.name, install_path)
chmod(install_path, 0755)
temp_f.close()
remove(temp_f.name)
log.d(TAG, "Copy complete!")
return 0
def auto_parse_module(self, args):
"""Automatically parse module and return Item"""
item = None
name = args.single_name
install_name = args.single_install_name
local_name = args.single_local_name
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = name
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = name
# Does the resource even exist?
if not os.path.isfile(local_name):
log.e(TAG, "Local module resource '%s' does not exist!"
% (local_name))
return None
if packagemanager.is_python_module(local_name, install_name):
log.d(TAG, "Python mode selected")
item = packagemanager.parse_python_module(local_name,
install_name)
if item is None:
log.e(TAG, "Error parsing Python module!")
return None
elif packagemanager.is_bash_module(local_name):
log.d(TAG, "Bash mode selected")
item = packagemanager.parse_bash_module(local_name,
install_name)
if item is None:
log.e(TAG, "Error parsing Bash module!")
return None
else:
log.e(TAG, "Auto parse for Python and Bash failed!")
return None
return item
def auto_parse_module(cls, args):
"""Automatically parse module and return Item"""
item = None
name = args.single_name
install_name = args.single_install_name
local_name = args.single_local_name
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = os.path.basename(name)
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = name
# Does the resource even exist?
if not os.path.isfile(local_name):
log.e(TAG,
"Local module resource '%s' does not exist!" % (local_name))
return None
if packagemanager.is_python_module(local_name, install_name):
log.d(TAG, "Python mode selected")
item = packagemanager.parse_python_module(local_name, install_name)
if item is None:
log.e(TAG, "Error parsing Python module!")
return None
elif packagemanager.is_bash_module(local_name):
log.d(TAG, "Bash mode selected")
item = packagemanager.parse_bash_module(local_name, install_name)
if item is None:
log.e(TAG, "Error parsing Bash module!")
return None
else:
log.e(TAG, "Auto parse for Python and Bash failed!")
return None
return item
def __install_zip_binary(zip_file, item, force_mode):
"""Install a binary from a ZIP"""
rtn = 0
# Does the resource even exist?
if not file_in_zip(zip_file, item.local_name):
log.w(
TAG, "'%s' defined, but local file '%s' does not exist!" %
(item.name, item.local_name))
return -1
try:
# First check if we know about this binary
if is_binary_installed(item.name):
# Prompt for install.
if not force_mode:
print(
"[WARNING] An item with this name is already installed."
" See details below.")
installed_item = __load_item(item)
if __prompt_install(item, installed_item):
log.d(TAG, "User would like to install")
rtn = __do_zip_binary_install(zip_file, item)
else:
log.i(TAG, "Binary installation skipped.")
# Force
else:
log.d(TAG, "Forcing component installation.")
rtn = __do_zip_binary_install(zip_file, item)
else:
log.d(TAG, "This is a new binary, installing.")
rtn = __do_zip_binary_install(zip_file, item)
except KeyError:
log.w(TAG, "Error checking if the binary was installed. Skipping")
rtn = -4
return rtn
def __install_zip_binary(zip_file, item, force_mode):
"""Install a binary from a ZIP"""
rtn = 0
# Does the resource even exist?
if not file_in_zip(zip_file, item.local_name):
log.w(TAG, "'%s' defined, but local file '%s' does not exist!"
% (item.name, item.local_name))
return -1
try:
# First check if we know about this binary
if is_binary_installed(item.name):
# Prompt for install.
if not force_mode:
print ("[WARNING] An item with this name is already installed."
" See details below.")
installed_item = __load_item(item)
if __prompt_install(item, installed_item):
log.d(TAG, "User would like to install")
rtn = __do_zip_binary_install(zip_file, item)
else:
log.i(TAG, "Binary installation skipped.")
# Force
else:
log.d(TAG, "Forcing component installation.")
rtn = __do_zip_binary_install(zip_file, item)
else:
log.d(TAG, "This is a new binary, installing.")
rtn = __do_zip_binary_install(zip_file, item)
except KeyError:
log.w(TAG, "Error checking if the binary was installed. Skipping")
rtn = -4
return rtn
def add_export_content(self, export_items, export_zip):
"""Add content to our ZIP file"""
for item in export_items:
if item.type == TYPE_LIBRARY or item.type == TYPE_PACKAGE:
for root, dirs, files in os.walk(item.install_name):
for dir_name in dirs:
file_path = os.path.join(root, dir_name)
rel_path = os.path.relpath(os.path.join(root, dir_name),
item.install_name)
zip_path = os.path.join(item.local_name, rel_path)
log.d(TAG, "Adding dir '%s' as '%s'"
% (file_path, zip_path))
export_zip.write(file_path, zip_path)
for file_name in files:
file_path = os.path.join(root, file_name)
rel_path = os.path.relpath(
os.path.join(root, file_name),
item.install_name)
zip_path = os.path.join(item.local_name, rel_path)
log.d(TAG, "Adding '%s' as '%s'"
% (file_path, zip_path))
export_zip.write(file_path, zip_path)
else:
log.d(TAG, "Adding '%s' as '%s'"
% (item.install_name, item.local_name))
export_zip.write(item.install_name, item.local_name)
return 0
def add_export_content(self, export_items, export_zip):
"""Add content to our ZIP file"""
for item in export_items:
if item.type == TYPE_LIBRARY or item.type == TYPE_PACKAGE:
for root, dirs, files in os.walk(item.install_name):
for dir_name in dirs:
file_path = os.path.join(root, dir_name)
rel_path = os.path.relpath(
os.path.join(root, dir_name), item.install_name)
zip_path = os.path.join(item.local_name, rel_path)
log.d(
TAG,
"Adding dir '%s' as '%s'" % (file_path, zip_path))
export_zip.write(file_path, zip_path)
for file_name in files:
file_path = os.path.join(root, file_name)
rel_path = os.path.relpath(
os.path.join(root, file_name), item.install_name)
zip_path = os.path.join(item.local_name, rel_path)
log.d(TAG,
"Adding '%s' as '%s'" % (file_path, zip_path))
export_zip.write(file_path, zip_path)
else:
log.d(
TAG, "Adding '%s' as '%s'" %
(item.install_name, item.local_name))
export_zip.write(item.install_name, item.local_name)
return 0
def __do_execute(self, command_string):
"""Do file execute"""
response = None
# Get a connected socket
sock = self.__sock_connect(DTF_SOCKET)
if sock is None:
log.e(TAG, "Cannot __do_execute, socket failure.")
return ("", ERR_SOCK)
sock.send(CMD_EXECUTE)
resp_code = self.__safe_recv(sock, 1, response=RESP_ERROR)
if resp_code != RESP_OK:
log.e(TAG, "Server rejected execute request!")
return (response, resp_code)
full_command = command_string.ljust(SIZE_CMD, '\0')
log.d(TAG, "Sending execute string to server")
sock.send(full_command)
log.d(TAG, "Command sent.")
binary_cmd_size = sock.recv(SIZE_INTEGER)
# This is an error.
if len(binary_cmd_size) == 1:
return (response, binary_cmd_size)
int_cmd_size = bytes_to_int(binary_cmd_size)
sock.send(RESP_OK)
if int_cmd_size == 0:
log.d(TAG, "Response is empty string!")
return ("", RESP_OK)
bytes_left = int_cmd_size
response = ""
transfer_success = False
while True:
if bytes_left <= SIZE_TRANSFER:
local_buf = self.__safe_recv(sock, bytes_left)
if local_buf is None:
break
response += local_buf
transfer_success = True
break
else:
local_buf = self.__safe_recv(sock, SIZE_TRANSFER)
if local_buf is None:
break
response += local_buf
bytes_left -= SIZE_TRANSFER
if not transfer_success:
log.e(TAG, "Error downloading file!")
return ("", RESP_ERROR)
sock.send(RESP_OK)
log.d(TAG, "Command complete!")
return (response, RESP_OK)
def initialize_device(self, device_serial):
"""Perform the actual initialization"""
log.d(TAG, "Preparing device: %s" % device_serial)
touch(utils.CONFIG_FILE_NAME)
set_prop('Info', 'serial', device_serial)
# Since we have a serial now, lets create a new DtfAdb instance
self.adb = DtfAdb.DtfAdb()
# Kernel
self.adb.shell_command('cat /proc/version')
kernel = self.adb.get_output()[0]
log.d(TAG, "Kernel version: %s" % kernel)
set_prop('Info', 'kernel', kernel)
# SDK
sdk = self.getprop('ro.build.version.sdk')
log.d(TAG, "Using SDK API %s" % sdk)
set_prop('Info', 'SDK', sdk)
self.adb.shell_command('set')
set_output = self.adb.get_output()
# $PATH
path = get_set_value(set_output, 'PATH')
if path is None:
log.e(TAG, "Unable to get $PATH variable!")
self.do_shutdown(None, None)
log.d(TAG, "PATH : %s" % path)
set_prop('Info', 'path', path)
# $BOOTCLASSPTH
bootclasspath = get_set_value(set_output, 'BOOTCLASSPATH')
if bootclasspath is None:
log.e(TAG, "Unable to get $BOOTCLASSPATH variable!")
self.do_shutdown(None, None)
log.d(TAG, "BOOTCLASSPATH : %s" % bootclasspath)
set_prop('Info', 'bootclasspath-jars', bootclasspath)
# Version string
version_string = self.generate_version_string()
log.d(TAG, "Using version string: %s" % version_string)
set_prop('Info', 'version-string', version_string)
# Determine CPU bitness
cpu_bits = self.determine_cpu_bits()
if cpu_bits is None:
self.do_shutdown(None, None)
log.d(TAG, "Using %s-bit CPU" % cpu_bits)
set_prop('Info', 'cpu-bits', cpu_bits)
# Set the VM type (Dalvik|Art)
vm_type = self.determine_vm_type(sdk, cpu_bits)
if vm_type is None:
self.do_shutdown(None, None)
log.d(TAG, "Determined runtime: %s" % vm_type)
set_prop('Info', 'vmtype', vm_type)
# Make project directories
mkdir(REPORTS_DIRECTORY)
mkdir(DBS_DIRECTORY)
mkdir(LOCAL_MODULES_DIRECTORY)
set_prop('Local', 'reports-dir', REPORTS_DIRECTORY)
set_prop('Local', 'db-dir', DBS_DIRECTORY)
# Invoke client installation
rtn = pkg.launch_builtin_module('client', ['install'])
if rtn != 0:
log.w(TAG, "Unable to install dtf client. Try manually.")
return 0
def purge():
"""Perform database purge"""
log.i(TAG, "Starting purge....")
dtf_db = sqlite3.connect(DTF_DB)
cur = dtf_db.cursor()
# Remove Binaries
sql = ('SELECT name, install_name '
'FROM binaries')
for row in cur.execute(sql):
binary_name = row[0]
install_name = row[1]
full_path = DTF_BINARIES_DIR + install_name
log.d(TAG, "Removing binary '%s'" % binary_name)
if delete_file(full_path) != 0:
log.e(TAG, "Error removing binary file! Continuing.")
# Remove Libraries
sql = ('SELECT name, install_name '
'FROM libraries')
for row in cur.execute(sql):
library_name = row[0]
install_name = row[1]
full_path = DTF_LIBRARIES_DIR + install_name
log.d(TAG, "Removing library '%s'" % library_name)
if delete_tree(full_path) != 0:
log.e(TAG, "Error removing library! Continuing.")
# Remove Modules
sql = ('SELECT name, install_name '
'FROM modules')
for row in cur.execute(sql):
module_name = row[0]
install_name = row[1]
full_path = DTF_MODULES_DIR + install_name
log.d(TAG, "Removing module '%s'" % module_name)
if delete_file(full_path) != 0:
log.e(TAG, "Error removing module file! Continuing.")
# Remove Packages
sql = ('SELECT name, install_name '
'FROM packages')
for row in cur.execute(sql):
package_name = row[0]
install_name = row[1]
full_path = DTF_PACKAGES_DIR + install_name
log.d(TAG, "Removing package '%s'" % package_name)
if delete_tree(full_path) != 0:
log.e(TAG, "Error removing package! Continuing.")
# Drop the DB.
cur.execute("DROP TABLE IF EXISTS binaries")
cur.execute("DROP TABLE IF EXISTS libraries")
cur.execute("DROP TABLE IF EXISTS modules")
cur.execute("DROP TABLE IF EXISTS packages")
dtf_db.commit()
# Rebuilding
if initialize_db() != 0:
log.e(TAG, "Unable to re-create dtf db!")
return -1
log.i(TAG, "Purge complete!")
return 0
def parse_single_item(self, args):
"""Parse args, return Item"""
item = packagemanager.Item()
name = args.single_name
if name is None:
log.e(TAG, "No '--name' specified in single item mode. Exiting.")
return None
item.name = name
single_type = args.single_type
if single_type not in packagemanager.VALID_TYPES:
log.e(TAG, "Invalid type passed to single. Exiting.")
return None
item.type = single_type
health = args.single_health
if health not in packagemanager.VALID_HEALTH_VALUES:
log.e(TAG, "Invalid health specified. Exiting.")
return None
item.health = health
version = args.single_version
if version is not None:
try:
(item.major_version, item.minor_version) = version.split('.')
except ValueError:
log.e(TAG, "Version string is not valid. Exiting.")
return None
else:
item.major_version = None
item.minor_version = None
try:
item.author = " ".join(args.single_author)
except TypeError:
item.author = None
try:
item.about = " ".join(args.single_about)
except TypeError:
item.about = None
install_name = args.single_install_name
local_name = args.single_local_name
if install_name is None:
log.d(TAG, "install_name is null, using name...")
install_name = name
if local_name is None:
log.d(TAG, "local_name is null, using name...")
local_name = name
item.install_name = install_name
item.local_name = local_name
if item.type == TYPE_BINARY:
if not os.path.isfile(item.local_name):
log.e(TAG, "Local item '%s' does not exist. Exiting."
% (item.local_name))
return None
elif item.type == TYPE_LIBRARY:
if not os.path.isdir(item.local_name):
log.e(TAG, "Local directory '%s' does not exist. Exiting."
% (item.local_name))
return None
elif item.type == TYPE_MODULE:
if not os.path.isfile(item.local_name):
log.e(TAG, "Local item '%s' does not exist. Exiting."
% (item.local_name))
return None
elif item.type == TYPE_PACKAGE:
if not os.path.isdir(item.local_name):
log.e(TAG, "Local directory '%s' does not exist. Exiting."
% (item.local_name))
return None
return item
def __sock_connect(cls,
socket_name,
socket_family=socket.AF_UNIX,
socket_type=socket.SOCK_STREAM):
""" Connect to socket_name.
First try abstract and fall back to filesystem
"""
# Create an unbound and not-connected socket.
try:
sock = socket.socket(socket_family, socket_type)
except socket.error as err:
log.e(TAG, "Socket creation failed: " + err.message)
return None
try:
log.d(TAG, "Connecting to abstract socket...")
# \0 denotes an abstract socket
sock.connect('\0' + socket_name)
except socket.error:
# abstract socket connection failed - it probably doesn't exist
# see jakev/dtf GitHub Issue #35
log.d(TAG, "Connecting to abstract socket failed. Does it exist?")
try:
log.d(TAG, "Connecting to filesystem socket...")
sock.connect('/tmp/' + socket_name)
except socket.error as err:
log.d(TAG,
"Connecting to filesystem socket failed: " + err.message)
log.e(TAG, "Connecting to socket failed, giving up.")
return None
else:
log.d(TAG, "Connected to filesystem socket!")
else:
log.d(TAG, "Connected to abstract socket!")
return sock