def run(self, path_arg=""):
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Cleaning {}".format(lib.name))
found = True
if os.path.exists(lib.build_dir):
try:
shutil.rmtree(lib.build_dir)
except Exception:
cls, value, traceback = sys.exc_info()
if cls == OSError and 'No such file or directory' in str(
value
): # TODO see if there is a proper way to do this
pass # Nothing to do here, the directory didn't exist
else:
logger.warning(
'Failed to clean the build directory: {}: {}'.
format(type, value))
else:
logger.warning(
'Nothing to clean. Path does not exist: {}'.format(
lib.build_dir))
if single_lib and not found:
raise ClickableException(
'Cannot clean unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def run(self, path_arg=None):
devices = self.device.detect_attached()
if len(devices) == 0:
logger.warning('No attached devices')
else:
for device in devices:
logger.info(device)
def build(self):
if os.path.isdir(self.config.install_dir):
raise ClickableException(
'Build directory already exists. Please run "clickable clean" before building again!'
)
shutil.copytree(self.config.cwd,
self.config.install_dir,
ignore=self._ignore)
logger.info('Copied files to install directory for click building')
def run(self, path_arg=''):
if not requests_available:
raise ClickableException(
'Unable to publish app, python requests module is not installed'
)
if not self.config.apikey:
raise ClickableException(
'No api key specified, use OPENSTORE_API_KEY or --apikey')
click = self.config.install_files.get_click_filename()
click_path = os.path.join(self.config.build_dir, click)
url = OPENSTORE_API
if 'OPENSTORE_API' in os.environ and os.environ['OPENSTORE_API']:
url = os.environ['OPENSTORE_API']
package_name = self.config.install_files.find_package_name()
url = url + OPENSTORE_API_PATH.format(package_name)
channel = 'xenial'
files = {'file': open(click_path, 'rb')}
data = {
'channel': channel,
'changelog': path_arg.encode('utf8', 'surrogateescape'),
}
params = {'apikey': self.config.apikey}
logger.info(
'Uploading version {} of {} for {}/{} to the OpenStore'.format(
self.config.install_files.find_version(),
package_name,
channel,
self.config.arch,
))
response = requests.post(url, files=files, data=data, params=params)
if response.status_code == requests.codes.ok:
logger.info('Upload successful')
elif response.status_code == requests.codes.not_found:
title = urllib.parse.quote(
self.config.install_files.find_package_title())
raise ClickableException(
'App needs to be created in the OpenStore before you can publish it. Visit {}/submit?appId={}&name={}'
.format(
OPENSTORE_API,
package_name,
title,
))
else:
if response.text == 'Unauthorized':
raise ClickableException(
'Failed to upload click: Unauthorized')
else:
raise ClickableException('Failed to upload click: {}'.format(
response.json()['message']))
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Building {}".format(lib.name))
found = True
lib.container_mode = self.config.container_mode
lib.docker_image = self.config.docker_image
lib.build_arch = self.config.build_arch
lib.container = Container(lib, lib.name)
lib.container.setup()
# This is a workaround for lib env vars being overwritten by
# project env vars, especially affecting Container Mode.
lib.set_env_vars()
try:
os.makedirs(lib.build_dir, exist_ok=True)
except Exception:
logger.warning(
'Failed to create the build directory: {}'.format(
str(sys.exc_info()[0])))
try:
os.makedirs(lib.build_home, exist_ok=True)
except Exception:
logger.warning(
'Failed to create the build home directory: {}'.format(
str(sys.exc_info()[0])))
if lib.prebuild:
run_subprocess_check_call(lib.prebuild,
cwd=self.config.cwd,
shell=True)
self.build(lib)
if lib.postbuild:
run_subprocess_check_call(lib.postbuild,
cwd=lib.build_dir,
shell=True)
if single_lib and not found:
raise ClickableException(
'Cannot build unknown library {}, which is not in your clickable.json'
.format(single_lib))
def start_docker(self):
started = False
error_code = run_subprocess_call(shlex.split('which systemctl'),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if error_code == 0:
logger.info('Asking for root to start docker')
error_code = run_subprocess_call(
shlex.split('sudo systemctl start docker'))
started = (error_code == 0)
return started
def setup_volume_mappings(self, local_working_directory, package_name):
xauth_path = self.touch_xauth()
device_home = self.config.desktop_device_home
makedirs(device_home)
logger.info("Mounting device home to {}".format(device_home))
return {
local_working_directory: local_working_directory,
'/tmp/.X11-unix': '/tmp/.X11-unix',
xauth_path: xauth_path,
device_home: '/home/phablet',
'/etc/timezone': '/etc/timezone',
}
def install_files(self, pattern, dest_dir):
if not is_sub_dir(dest_dir, self.config.install_dir):
dest_dir = os.path.abspath(self.config.install_dir + "/" + dest_dir)
makedirs(dest_dir)
if '"' in pattern:
# Make sure one cannot run random bash code through the "ls" command
raise ClickableException("install_* patterns must not contain any '\"' quotation character.")
command = 'ls -d "{}"'.format(pattern)
files = self.config.container.run_command(command, get_output=True).split()
logger.info("Installing {}".format(", ".join(files)))
self.config.container.pull_files(files, dest_dir)
def run(self, path_arg=None):
if self.config.is_desktop_mode():
logger.debug('Skipping install, running in desktop mode')
return
elif self.config.container_mode:
logger.debug('Skipping install, running in container mode')
return
cwd = '.'
if path_arg:
click = os.path.basename(path_arg)
click_path = path_arg
else:
click = self.config.install_files.get_click_filename()
click_path = os.path.join(self.config.build_dir, click)
cwd = self.config.build_dir
if self.config.ssh:
command = 'scp {} phablet@{}:/home/phablet/'.format(
click_path, self.config.ssh)
run_subprocess_check_call(command, cwd=cwd, shell=True)
else:
self.device.check_any_attached()
if self.config.device_serial_number:
command = 'adb -s {} push {} /home/phablet/'.format(
self.config.device_serial_number, click_path)
else:
self.device.check_multiple_attached()
command = 'adb push {} /home/phablet/'.format(click_path)
run_subprocess_check_call(command, cwd=cwd, shell=True)
if path_arg:
logger.info(
"Skipping uninstall step, because you specified a click package."
)
else:
self.try_uninstall()
self.device.run_command(
'pkcon install-local --allow-untrusted /home/phablet/{}'.format(
click),
cwd=cwd)
self.device.run_command('rm /home/phablet/{}'.format(click), cwd=cwd)
def before_run(self, config, docker_config):
#if first qtcreator launch, install common settings
if not os.path.isdir(self.target_settings_path):
logger.info('copy initial qtcreator settings to {}'.format(
self.clickable_dir))
tar = tarfile.open(self.init_settings_path)
tar.extractall(self.clickable_dir)
tar.close()
if self.is_cmake_project() and not os.path.isfile(
os.path.join(self.project_path, 'CMakeLists.txt.user')):
self.init_cmake_project(config, docker_config)
#delete conflicting env vars in some cases
docker_config.environment.pop("INSTALL_DIR", None)
docker_config.environment.pop("APP_DIR", None)
docker_config.environment.pop("SRC_DIR", None)
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Running tests on {}".format(lib.name))
found = True
self.run_test(lib)
if single_lib and not found:
raise ClickableException(
'Cannot test unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Building {}".format(lib.name))
found = True
lib.container_mode = self.config.container_mode
lib.docker_image = self.config.docker_image
lib.build_arch = self.config.build_arch
lib.container = Container(lib, lib.name)
lib.container.setup_dependencies()
try:
os.makedirs(lib.build_dir)
except FileExistsError:
pass
except Exception:
logger.warning(
'Failed to create the build directory: {}'.format(
str(sys.exc_info()[0])))
if lib.prebuild:
run_subprocess_check_call(lib.prebuild,
cwd=self.config.cwd,
shell=True)
self.build(lib)
if lib.postbuild:
run_subprocess_check_call(lib.postbuild,
cwd=lib.build_dir,
shell=True)
if single_lib and not found:
raise ClickableException(
'Cannot build unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def setup_volume_mappings(self):
xauth_path = self.touch_xauth()
device_home = Constants.desktop_device_home
makedirs(device_home)
logger.info("Mounting device home to {}".format(device_home))
vol_map = {
self.config.cwd: self.config.cwd,
'/tmp/.X11-unix': '/tmp/.X11-unix',
xauth_path: xauth_path,
device_home: '/home/phablet',
'/etc/passwd': '/etc/passwd',
}
if self.custom_mode:
user_home = os.path.expanduser('~')
vol_map[user_home] = user_home
return vol_map
def main():
clickable = Clickable()
args = clickable.parse_args()
if args.verbose:
console_handler.setLevel(logging.DEBUG)
logger.debug('Clickable v' + __version__)
clickable.check_version(quiet=True)
try:
clickable.run(args.commands, args)
except ClickableException as e:
logger.error(str(e))
sys.exit(1)
except subprocess.CalledProcessError as e:
logger.debug('Command exited with an error:' + str(e.cmd), exc_info=e)
logger.critical(
'Command exited with non-zero exit status {}, see above for details. This is most likely not a problem with Clickable.'
.format(e.returncode, ))
sys.exit(2)
except KeyboardInterrupt as e:
logger.info(
'') # Print an empty space at then end so the cli prompt is nicer
sys.exit(0)
except Exception as e:
if isinstance(e, OSError) and '28' in str(e):
logger.critical('No space left on device')
sys.exit(2)
return
logger.debug('Encountered an unknown error', exc_info=e)
if not args.verbose:
logger.critical('Encountered an unknown error: ' + str(e))
logger.critical(
'If you believe this is a bug, please file a report at https://gitlab.com/clickable/clickable/issues with the log file located at '
+ log_file)
sys.exit(3)
def setup_docker(self):
logger.info('Setting up docker')
self.start_docker()
if not self.docker_group_exists():
logger.info('Asking for root to create docker group')
subprocess.check_call(shlex.split('sudo groupadd docker'))
if self.user_part_of_docker_group():
logger.info('Setup has already been completed')
else:
logger.info(
'Asking for root to add the current user to the docker group')
subprocess.check_call(
shlex.split('sudo usermod -aG docker {}'.format(
getpass.getuser())))
raise ClickableException('Log out or restart to apply changes')
def run_command(self,
command,
root_user=False,
get_output=False,
use_build_dir=True,
cwd=None,
tty=False,
localhost=False):
wrapped_command = command
cwd = cwd if cwd else os.path.abspath(self.config.root_dir)
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
else: # Docker
self.check_docker()
if ' ' in cwd or ' ' in self.config.build_dir:
raise ClickableException(
'There are spaces in the current path, this will cause errors in the build process'
)
if self.config.first_docker_info:
logger.debug('Using docker container "{}"'.format(
self.docker_image))
self.config.first_docker_info = False
go_config = ''
if self.config.builder == Constants.GO and self.config.gopath:
gopaths = self.config.gopath.split(':')
docker_gopaths = [
'/gopath/path{}'.format(index)
for index in range(len(gopaths))
]
go_config = '-e GOPATH={}'.format(':'.join(docker_gopaths), )
rust_config = ''
if self.config.builder == Constants.RUST and self.config.cargo_home:
logger.info("Caching cargo related files in {}".format(
self.config.cargo_home))
env_vars = self.config.prepare_docker_env_vars()
user = ""
if not root_user:
user = "******".format(os.getuid())
mounts = self.render_mounts(
self.get_docker_mounts(transparent=[cwd]))
wrapped_command = 'docker run {mounts} {env} {go} {rust} {user} -w {cwd} --rm {tty} {network} -i {image} bash -c "{cmd}"'.format(
mounts=mounts,
env=env_vars,
go=go_config,
rust=rust_config,
cwd=self.config.build_dir if use_build_dir else cwd,
user=user,
image=self.docker_image,
cmd=command,
tty="-t" if tty else "",
network='--network="host"' if localhost else "",
)
kwargs = {}
if use_build_dir:
kwargs['cwd'] = self.config.build_dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command),
**kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def show_version(self):
logger.info('clickable ' + __version__)
self.check_version()
def check_version(self, quiet=False):
if requests_available:
version = None
check = True
version_check = expanduser('~/.clickable/version_check.json')
if isfile(version_check):
with open(version_check, 'r') as f:
try:
version_check_data = json.load(f)
except ValueError:
version_check_data = None
if version_check_data and 'version' in version_check_data and 'datetime' in version_check_data:
last_check = datetime.strptime(
version_check_data['datetime'], DATE_FORMAT)
if last_check > (datetime.now() - timedelta(days=2)) and \
'current_version' in version_check_data and \
version_check_data['current_version'] == __version__:
check = False
version = version_check_data['version']
logger.debug('Using cached version check')
if check:
logger.debug('Checking for updates to clickable')
try:
response = requests.get(
'https://clickable-ut.dev/en/latest/_static/version.json',
timeout=5)
response.raise_for_status()
data = response.json()
version = data['version']
except requests.exceptions.Timeout as e:
logger.warning(
'Unable to check for updates to clickable, the request timedout'
)
except Exception as e:
logger.debug('Version check failed:' + str(e.cmd),
exc_info=e)
logger.warning(
'Unable to check for updates to clickable, an unknown error occurred'
)
if version:
with open(version_check, 'w') as f:
json.dump(
{
'version': version,
'datetime':
datetime.now().strftime(DATE_FORMAT),
'current_version': __version__,
}, f)
if version:
if version != __version__:
logger.info(
'v{} of clickable is available, update to get the latest features and improvements!'
.format(version))
else:
if not quiet:
logger.info(
'You are running the latest version of clickable!')
else:
if not quiet:
logger.warning(
'Unable to check for updates to clickable, please install "requests"'
)
def install_files(self, pattern, dest_dir):
logger.info("Installing {}".format(pattern))
makedirs(dest_dir)
command = 'cp -r {} {}'.format(pattern, dest_dir)
self.config.container.run_command(command)
def run(self, path_arg=None):
command = 'dbus-send --system --print-reply --dest=com.canonical.PropertyService /com/canonical/PropertyService com.canonical.PropertyService.SetProperty string:writable boolean:true'
self.device.run_command(command, cwd=self.config.cwd)
logger.info('Rebooting device for writable image')
def run(self, path_arg=None):
try:
self.config.container.run_command("echo ''", use_build_dir=False)
logger.info('Clickable is set up and ready.')
except ClickableException:
logger.warning('Please log out or restart to apply changes')
def run_command(self,
command,
sudo=False,
get_output=False,
use_dir=True,
cwd=None):
wrapped_command = command
cwd = cwd if cwd else os.path.abspath(self.config.root_dir)
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
else: # Docker
self.check_docker()
if ' ' in cwd or ' ' in self.config.build_dir:
raise ClickableException(
'There are spaces in the current path, this will cause errors in the build process'
)
if self.config.first_docker_info:
logger.debug('Using docker container "{}"'.format(
self.docker_image))
self.config.first_docker_info = False
go_config = ''
if self.config.gopath:
gopaths = self.config.gopath.split(':')
docker_gopaths = []
go_configs = []
for (index, path) in enumerate(gopaths):
go_configs.append('-v {}:/gopath/path{}:Z'.format(
path, index))
docker_gopaths.append('/gopath/path{}'.format(index))
go_config = '{} -e GOPATH={}'.format(
' '.join(go_configs),
':'.join(docker_gopaths),
)
rust_config = ''
if self.config.config[
'template'] == Config.RUST and self.config.cargo_home:
logger.info("Caching cargo related files in {}".format(
self.config.cargo_home))
cargo_registry = os.path.join(self.config.cargo_home,
'registry')
cargo_git = os.path.join(self.config.cargo_home, 'git')
cargo_package_cache_lock = os.path.join(
self.config.cargo_home, '.package-cache')
os.makedirs(cargo_registry, exist_ok=True)
os.makedirs(cargo_git, exist_ok=True)
# create .package-cache if it doesn't exist
with open(cargo_package_cache_lock, "a"):
pass
rust_config = '-v {}:/opt/rust/cargo/registry:Z -v {}:/opt/rust/cargo/git:Z -v {}:/opt/rust/cargo/.package-cache'.format(
cargo_registry,
cargo_git,
cargo_package_cache_lock,
)
env_vars = self.config.prepare_docker_env_vars()
wrapped_command = 'docker run -v {}:{}:Z {} {} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format(
cwd,
cwd,
env_vars,
go_config,
rust_config,
self.config.build_dir if use_dir else cwd,
os.getuid(),
self.docker_image,
command,
)
kwargs = {}
if use_dir:
kwargs['cwd'] = self.config.build_dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command),
**kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def run(self, path_arg=None):
'''
Inspired by http://bazaar.launchpad.net/~phablet-team/phablet-tools/trunk/view/head:/phablet-shell
'''
if self.config.ssh:
subprocess.check_call(
shlex.split('ssh phablet@{}'.format(self.config.ssh)))
else:
self.device.check_any_attached()
adb_args = ''
if self.config.device_serial_number:
adb_args = '-s {}'.format(self.config.device_serial_number)
else:
self.device.check_multiple_attached()
output = run_subprocess_check_output(
shlex.split(
'adb {} shell pgrep sshd'.format(adb_args))).split()
if not output:
self.toggle_ssh(on=True)
# Use the usb cable rather than worrying about going over wifi
port = 0
for p in range(2222, 2299):
error_code = run_subprocess_call(shlex.split(
'adb {} forward tcp:{} tcp:22'.format(adb_args, p)),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if error_code == 0:
port = p
break
if port == 0:
raise ClickableException('Failed to open a port to the device')
# Purge the device host key so that SSH doesn't print a scary warning about it
# (it changes every time the device is reflashed and this is expected)
known_hosts = os.path.expanduser('~/.ssh/known_hosts')
subprocess.check_call(shlex.split('touch {}'.format(known_hosts)))
subprocess.check_call(
shlex.split('ssh-keygen -f {} -R [localhost]:{}'.format(
known_hosts, port)))
id_pub = os.path.expanduser('~/.ssh/id_rsa.pub')
if not os.path.isfile(id_pub):
raise ClickableException(
'Could not find a ssh public key at "{}", please generate one and try again'
.format(id_pub))
with open(id_pub, 'r') as f:
public_key = f.read().strip()
self.device.run_command('[ -d ~/.ssh ] || mkdir ~/.ssh',
cwd=self.config.cwd)
self.device.run_command('touch ~/.ssh/authorized_keys',
cwd=self.config.cwd)
output = run_subprocess_check_output(
'adb {} shell "grep \\"{}\\" ~/.ssh/authorized_keys"'.format(
adb_args, public_key),
shell=True).strip()
if not output or 'No such file or directory' in output:
logger.info('Inserting ssh public key on the connected device')
self.device.run_command(
'echo \"{}\" >>~/.ssh/authorized_keys'.format(public_key),
cwd=self.config.cwd)
self.device.run_command('chmod 700 ~/.ssh',
cwd=self.config.cwd)
self.device.run_command('chmod 600 ~/.ssh/authorized_keys',
cwd=self.config.cwd)
subprocess.check_call(
shlex.split(
'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p {} phablet@localhost'
.format(port)))
self.toggle_ssh(on=False)
def run(self, path_arg=None):
logger.info('Turning off device activity timeout')
command = 'gsettings set com.ubuntu.touch.system activity-timeout 0'
self.device.run_command(command, cwd=self.config.cwd)
def init_cmake_project(self, config, docker_config):
executable = config.project_files.find_any_executable()
exec_args = " ".join(config.project_files.find_any_exec_args())
#don't do all that if exec line not found
if not executable:
return
choice = input(
Colors.INFO +
'Do you want Clickable to setup a QtCreator project for you? [Y/n]: '
+ Colors.CLEAR).strip().lower()
if choice != 'y' and choice != 'yes' and choice != '':
return
#CLICK_EXE can be a variable
match_exe_var = re.match("@([-\w]+)@", executable)
if match_exe_var:
#catch the variable name and try to get it from CMakeLists.txt
cmd_var = match_exe_var.group(1)
final_cmd = self.cmake_guess_exec_command(cmd_var)
if final_cmd is not None:
try:
exe, exe_arg = final_cmd.split(' ', maxsplit=1)
except:
exe, exe_arg = final_cmd, ''
executable = exe
exec_args = exe_arg
logger.debug(
'found that executable is {} with args: {}'.format(
exe, exe_arg))
else:
#was not able to guess executable
logger.warning(
"Could not determine executable command '{}', please adjust your project's run settings"
.format(executable))
# work around for qtcreator bug when first run of a project to avoid qtcreator hang
# we need to create the build directory first
if not os.path.isdir(config.build_dir):
os.makedirs(config.build_dir)
env_vars = docker_config.environment
clickable_env_path = '{}:{}'.format(env_vars["PATH"],
env_vars["CLICK_PATH"])
clickable_ld_library_path = '{}:{}'.format(
env_vars["LD_LIBRARY_PATH"], env_vars["CLICK_LD_LIBRARY_PATH"])
clickable_qml2_import_path = '{}:{}:{}'.format(
env_vars["QML2_IMPORT_PATH"], env_vars["CLICK_QML2_IMPORT_PATH"],
os.path.join(config.install_dir, 'lib'))
template_replacement = {
"CLICKABLE_LD_LIBRARY_PATH": clickable_ld_library_path,
"CLICKABLE_QML2_IMPORT_PATH": clickable_qml2_import_path,
"CLICKABLE_BUILD_DIR": config.build_dir,
"CLICKABLE_INSTALL_DIR": config.install_dir,
"CLICKABLE_EXEC_CMD": executable,
"CLICKABLE_EXEC_ARGS": exec_args,
"CLICKABLE_SRC_DIR": config.src_dir,
"CLICKABLE_BUILD_ARGS": " ".join(config.build_args),
"CLICKABLE_PATH": clickable_env_path
}
output_path = os.path.join(self.project_path,
'CMakeLists.txt.user.shared')
#now read template and generate the .shared file to the root project dir
with open(self.template_path, "r") as infile2, open(output_path,
"w") as outfile:
for line in infile2:
for f_key, f_value in template_replacement.items():
if f_key in line:
line = line.replace(f_key, f_value)
outfile.write(line)
logger.info(
'generated default build/run template to {}'.format(output_path))
