def run(self, path_arg=""): single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Cleaning {}".format(lib.name)) found = True if os.path.exists(lib.build_dir): try: shutil.rmtree(lib.build_dir) except Exception: cls, value, traceback = sys.exc_info() if cls == OSError and 'No such file or directory' in str( value ): # TODO see if there is a proper way to do this pass # Nothing to do here, the directory didn't exist else: logger.warning( 'Failed to clean the build directory: {}: {}'. format(type, value)) else: logger.warning( 'Nothing to clean. Path does not exist: {}'.format( lib.build_dir)) if single_lib and not found: raise ClickableException( 'Cannot clean unknown library {}. You may add it to the clickable.json' .format(single_lib))
def run(self, path_arg=None): devices = self.device.detect_attached() if len(devices) == 0: logger.warning('No attached devices') else: for device in devices: logger.info(device)
def build(self): if os.path.isdir(self.config.install_dir): raise ClickableException( 'Build directory already exists. Please run "clickable clean" before building again!' ) shutil.copytree(self.config.cwd, self.config.install_dir, ignore=self._ignore) logger.info('Copied files to install directory for click building')
def run(self, path_arg=''): if not requests_available: raise ClickableException( 'Unable to publish app, python requests module is not installed' ) if not self.config.apikey: raise ClickableException( 'No api key specified, use OPENSTORE_API_KEY or --apikey') click = self.config.install_files.get_click_filename() click_path = os.path.join(self.config.build_dir, click) url = OPENSTORE_API if 'OPENSTORE_API' in os.environ and os.environ['OPENSTORE_API']: url = os.environ['OPENSTORE_API'] package_name = self.config.install_files.find_package_name() url = url + OPENSTORE_API_PATH.format(package_name) channel = 'xenial' files = {'file': open(click_path, 'rb')} data = { 'channel': channel, 'changelog': path_arg.encode('utf8', 'surrogateescape'), } params = {'apikey': self.config.apikey} logger.info( 'Uploading version {} of {} for {}/{} to the OpenStore'.format( self.config.install_files.find_version(), package_name, channel, self.config.arch, )) response = requests.post(url, files=files, data=data, params=params) if response.status_code == requests.codes.ok: logger.info('Upload successful') elif response.status_code == requests.codes.not_found: title = urllib.parse.quote( self.config.install_files.find_package_title()) raise ClickableException( 'App needs to be created in the OpenStore before you can publish it. Visit {}/submit?appId={}&name={}' .format( OPENSTORE_API, package_name, title, )) else: if response.text == 'Unauthorized': raise ClickableException( 'Failed to upload click: Unauthorized') else: raise ClickableException('Failed to upload click: {}'.format( response.json()['message']))
def run(self, path_arg=""): if not self.config.lib_configs: logger.warning('No libraries defined.') single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Building {}".format(lib.name)) found = True lib.container_mode = self.config.container_mode lib.docker_image = self.config.docker_image lib.build_arch = self.config.build_arch lib.container = Container(lib, lib.name) lib.container.setup() # This is a workaround for lib env vars being overwritten by # project env vars, especially affecting Container Mode. lib.set_env_vars() try: os.makedirs(lib.build_dir, exist_ok=True) except Exception: logger.warning( 'Failed to create the build directory: {}'.format( str(sys.exc_info()[0]))) try: os.makedirs(lib.build_home, exist_ok=True) except Exception: logger.warning( 'Failed to create the build home directory: {}'.format( str(sys.exc_info()[0]))) if lib.prebuild: run_subprocess_check_call(lib.prebuild, cwd=self.config.cwd, shell=True) self.build(lib) if lib.postbuild: run_subprocess_check_call(lib.postbuild, cwd=lib.build_dir, shell=True) if single_lib and not found: raise ClickableException( 'Cannot build unknown library {}, which is not in your clickable.json' .format(single_lib))
def start_docker(self): started = False error_code = run_subprocess_call(shlex.split('which systemctl'), stdout=subprocess.PIPE, stderr=subprocess.PIPE) if error_code == 0: logger.info('Asking for root to start docker') error_code = run_subprocess_call( shlex.split('sudo systemctl start docker')) started = (error_code == 0) return started
def setup_volume_mappings(self, local_working_directory, package_name): xauth_path = self.touch_xauth() device_home = self.config.desktop_device_home makedirs(device_home) logger.info("Mounting device home to {}".format(device_home)) return { local_working_directory: local_working_directory, '/tmp/.X11-unix': '/tmp/.X11-unix', xauth_path: xauth_path, device_home: '/home/phablet', '/etc/timezone': '/etc/timezone', }
def install_files(self, pattern, dest_dir): if not is_sub_dir(dest_dir, self.config.install_dir): dest_dir = os.path.abspath(self.config.install_dir + "/" + dest_dir) makedirs(dest_dir) if '"' in pattern: # Make sure one cannot run random bash code through the "ls" command raise ClickableException("install_* patterns must not contain any '\"' quotation character.") command = 'ls -d "{}"'.format(pattern) files = self.config.container.run_command(command, get_output=True).split() logger.info("Installing {}".format(", ".join(files))) self.config.container.pull_files(files, dest_dir)
def run(self, path_arg=None): if self.config.is_desktop_mode(): logger.debug('Skipping install, running in desktop mode') return elif self.config.container_mode: logger.debug('Skipping install, running in container mode') return cwd = '.' if path_arg: click = os.path.basename(path_arg) click_path = path_arg else: click = self.config.install_files.get_click_filename() click_path = os.path.join(self.config.build_dir, click) cwd = self.config.build_dir if self.config.ssh: command = 'scp {} phablet@{}:/home/phablet/'.format( click_path, self.config.ssh) run_subprocess_check_call(command, cwd=cwd, shell=True) else: self.device.check_any_attached() if self.config.device_serial_number: command = 'adb -s {} push {} /home/phablet/'.format( self.config.device_serial_number, click_path) else: self.device.check_multiple_attached() command = 'adb push {} /home/phablet/'.format(click_path) run_subprocess_check_call(command, cwd=cwd, shell=True) if path_arg: logger.info( "Skipping uninstall step, because you specified a click package." ) else: self.try_uninstall() self.device.run_command( 'pkcon install-local --allow-untrusted /home/phablet/{}'.format( click), cwd=cwd) self.device.run_command('rm /home/phablet/{}'.format(click), cwd=cwd)
def before_run(self, config, docker_config): #if first qtcreator launch, install common settings if not os.path.isdir(self.target_settings_path): logger.info('copy initial qtcreator settings to {}'.format( self.clickable_dir)) tar = tarfile.open(self.init_settings_path) tar.extractall(self.clickable_dir) tar.close() if self.is_cmake_project() and not os.path.isfile( os.path.join(self.project_path, 'CMakeLists.txt.user')): self.init_cmake_project(config, docker_config) #delete conflicting env vars in some cases docker_config.environment.pop("INSTALL_DIR", None) docker_config.environment.pop("APP_DIR", None) docker_config.environment.pop("SRC_DIR", None)
def run(self, path_arg=""): if not self.config.lib_configs: logger.warning('No libraries defined.') single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Running tests on {}".format(lib.name)) found = True self.run_test(lib) if single_lib and not found: raise ClickableException( 'Cannot test unknown library {}. You may add it to the clickable.json' .format(single_lib))
def run(self, path_arg=""): if not self.config.lib_configs: logger.warning('No libraries defined.') single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Building {}".format(lib.name)) found = True lib.container_mode = self.config.container_mode lib.docker_image = self.config.docker_image lib.build_arch = self.config.build_arch lib.container = Container(lib, lib.name) lib.container.setup_dependencies() try: os.makedirs(lib.build_dir) except FileExistsError: pass except Exception: logger.warning( 'Failed to create the build directory: {}'.format( str(sys.exc_info()[0]))) if lib.prebuild: run_subprocess_check_call(lib.prebuild, cwd=self.config.cwd, shell=True) self.build(lib) if lib.postbuild: run_subprocess_check_call(lib.postbuild, cwd=lib.build_dir, shell=True) if single_lib and not found: raise ClickableException( 'Cannot build unknown library {}. You may add it to the clickable.json' .format(single_lib))
def setup_volume_mappings(self): xauth_path = self.touch_xauth() device_home = Constants.desktop_device_home makedirs(device_home) logger.info("Mounting device home to {}".format(device_home)) vol_map = { self.config.cwd: self.config.cwd, '/tmp/.X11-unix': '/tmp/.X11-unix', xauth_path: xauth_path, device_home: '/home/phablet', '/etc/passwd': '/etc/passwd', } if self.custom_mode: user_home = os.path.expanduser('~') vol_map[user_home] = user_home return vol_map
def main(): clickable = Clickable() args = clickable.parse_args() if args.verbose: console_handler.setLevel(logging.DEBUG) logger.debug('Clickable v' + __version__) clickable.check_version(quiet=True) try: clickable.run(args.commands, args) except ClickableException as e: logger.error(str(e)) sys.exit(1) except subprocess.CalledProcessError as e: logger.debug('Command exited with an error:' + str(e.cmd), exc_info=e) logger.critical( 'Command exited with non-zero exit status {}, see above for details. This is most likely not a problem with Clickable.' .format(e.returncode, )) sys.exit(2) except KeyboardInterrupt as e: logger.info( '') # Print an empty space at then end so the cli prompt is nicer sys.exit(0) except Exception as e: if isinstance(e, OSError) and '28' in str(e): logger.critical('No space left on device') sys.exit(2) return logger.debug('Encountered an unknown error', exc_info=e) if not args.verbose: logger.critical('Encountered an unknown error: ' + str(e)) logger.critical( 'If you believe this is a bug, please file a report at https://gitlab.com/clickable/clickable/issues with the log file located at ' + log_file) sys.exit(3)
def setup_docker(self): logger.info('Setting up docker') self.start_docker() if not self.docker_group_exists(): logger.info('Asking for root to create docker group') subprocess.check_call(shlex.split('sudo groupadd docker')) if self.user_part_of_docker_group(): logger.info('Setup has already been completed') else: logger.info( 'Asking for root to add the current user to the docker group') subprocess.check_call( shlex.split('sudo usermod -aG docker {}'.format( getpass.getuser()))) raise ClickableException('Log out or restart to apply changes')
def run_command(self, command, root_user=False, get_output=False, use_build_dir=True, cwd=None, tty=False, localhost=False): wrapped_command = command cwd = cwd if cwd else os.path.abspath(self.config.root_dir) if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) else: # Docker self.check_docker() if ' ' in cwd or ' ' in self.config.build_dir: raise ClickableException( 'There are spaces in the current path, this will cause errors in the build process' ) if self.config.first_docker_info: logger.debug('Using docker container "{}"'.format( self.docker_image)) self.config.first_docker_info = False go_config = '' if self.config.builder == Constants.GO and self.config.gopath: gopaths = self.config.gopath.split(':') docker_gopaths = [ '/gopath/path{}'.format(index) for index in range(len(gopaths)) ] go_config = '-e GOPATH={}'.format(':'.join(docker_gopaths), ) rust_config = '' if self.config.builder == Constants.RUST and self.config.cargo_home: logger.info("Caching cargo related files in {}".format( self.config.cargo_home)) env_vars = self.config.prepare_docker_env_vars() user = "" if not root_user: user = "******".format(os.getuid()) mounts = self.render_mounts( self.get_docker_mounts(transparent=[cwd])) wrapped_command = 'docker run {mounts} {env} {go} {rust} {user} -w {cwd} --rm {tty} {network} -i {image} bash -c "{cmd}"'.format( mounts=mounts, env=env_vars, go=go_config, rust=rust_config, cwd=self.config.build_dir if use_build_dir else cwd, user=user, image=self.docker_image, cmd=command, tty="-t" if tty else "", network='--network="host"' if localhost else "", ) kwargs = {} if use_build_dir: kwargs['cwd'] = self.config.build_dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def show_version(self): logger.info('clickable ' + __version__) self.check_version()
def check_version(self, quiet=False): if requests_available: version = None check = True version_check = expanduser('~/.clickable/version_check.json') if isfile(version_check): with open(version_check, 'r') as f: try: version_check_data = json.load(f) except ValueError: version_check_data = None if version_check_data and 'version' in version_check_data and 'datetime' in version_check_data: last_check = datetime.strptime( version_check_data['datetime'], DATE_FORMAT) if last_check > (datetime.now() - timedelta(days=2)) and \ 'current_version' in version_check_data and \ version_check_data['current_version'] == __version__: check = False version = version_check_data['version'] logger.debug('Using cached version check') if check: logger.debug('Checking for updates to clickable') try: response = requests.get( 'https://clickable-ut.dev/en/latest/_static/version.json', timeout=5) response.raise_for_status() data = response.json() version = data['version'] except requests.exceptions.Timeout as e: logger.warning( 'Unable to check for updates to clickable, the request timedout' ) except Exception as e: logger.debug('Version check failed:' + str(e.cmd), exc_info=e) logger.warning( 'Unable to check for updates to clickable, an unknown error occurred' ) if version: with open(version_check, 'w') as f: json.dump( { 'version': version, 'datetime': datetime.now().strftime(DATE_FORMAT), 'current_version': __version__, }, f) if version: if version != __version__: logger.info( 'v{} of clickable is available, update to get the latest features and improvements!' .format(version)) else: if not quiet: logger.info( 'You are running the latest version of clickable!') else: if not quiet: logger.warning( 'Unable to check for updates to clickable, please install "requests"' )
def install_files(self, pattern, dest_dir): logger.info("Installing {}".format(pattern)) makedirs(dest_dir) command = 'cp -r {} {}'.format(pattern, dest_dir) self.config.container.run_command(command)
def run(self, path_arg=None): command = 'dbus-send --system --print-reply --dest=com.canonical.PropertyService /com/canonical/PropertyService com.canonical.PropertyService.SetProperty string:writable boolean:true' self.device.run_command(command, cwd=self.config.cwd) logger.info('Rebooting device for writable image')
def run(self, path_arg=None): try: self.config.container.run_command("echo ''", use_build_dir=False) logger.info('Clickable is set up and ready.') except ClickableException: logger.warning('Please log out or restart to apply changes')
def run_command(self, command, sudo=False, get_output=False, use_dir=True, cwd=None): wrapped_command = command cwd = cwd if cwd else os.path.abspath(self.config.root_dir) if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) else: # Docker self.check_docker() if ' ' in cwd or ' ' in self.config.build_dir: raise ClickableException( 'There are spaces in the current path, this will cause errors in the build process' ) if self.config.first_docker_info: logger.debug('Using docker container "{}"'.format( self.docker_image)) self.config.first_docker_info = False go_config = '' if self.config.gopath: gopaths = self.config.gopath.split(':') docker_gopaths = [] go_configs = [] for (index, path) in enumerate(gopaths): go_configs.append('-v {}:/gopath/path{}:Z'.format( path, index)) docker_gopaths.append('/gopath/path{}'.format(index)) go_config = '{} -e GOPATH={}'.format( ' '.join(go_configs), ':'.join(docker_gopaths), ) rust_config = '' if self.config.config[ 'template'] == Config.RUST and self.config.cargo_home: logger.info("Caching cargo related files in {}".format( self.config.cargo_home)) cargo_registry = os.path.join(self.config.cargo_home, 'registry') cargo_git = os.path.join(self.config.cargo_home, 'git') cargo_package_cache_lock = os.path.join( self.config.cargo_home, '.package-cache') os.makedirs(cargo_registry, exist_ok=True) os.makedirs(cargo_git, exist_ok=True) # create .package-cache if it doesn't exist with open(cargo_package_cache_lock, "a"): pass rust_config = '-v {}:/opt/rust/cargo/registry:Z -v {}:/opt/rust/cargo/git:Z -v {}:/opt/rust/cargo/.package-cache'.format( cargo_registry, cargo_git, cargo_package_cache_lock, ) env_vars = self.config.prepare_docker_env_vars() wrapped_command = 'docker run -v {}:{}:Z {} {} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format( cwd, cwd, env_vars, go_config, rust_config, self.config.build_dir if use_dir else cwd, os.getuid(), self.docker_image, command, ) kwargs = {} if use_dir: kwargs['cwd'] = self.config.build_dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def run(self, path_arg=None): ''' Inspired by http://bazaar.launchpad.net/~phablet-team/phablet-tools/trunk/view/head:/phablet-shell ''' if self.config.ssh: subprocess.check_call( shlex.split('ssh phablet@{}'.format(self.config.ssh))) else: self.device.check_any_attached() adb_args = '' if self.config.device_serial_number: adb_args = '-s {}'.format(self.config.device_serial_number) else: self.device.check_multiple_attached() output = run_subprocess_check_output( shlex.split( 'adb {} shell pgrep sshd'.format(adb_args))).split() if not output: self.toggle_ssh(on=True) # Use the usb cable rather than worrying about going over wifi port = 0 for p in range(2222, 2299): error_code = run_subprocess_call(shlex.split( 'adb {} forward tcp:{} tcp:22'.format(adb_args, p)), stdout=subprocess.PIPE, stderr=subprocess.PIPE) if error_code == 0: port = p break if port == 0: raise ClickableException('Failed to open a port to the device') # Purge the device host key so that SSH doesn't print a scary warning about it # (it changes every time the device is reflashed and this is expected) known_hosts = os.path.expanduser('~/.ssh/known_hosts') subprocess.check_call(shlex.split('touch {}'.format(known_hosts))) subprocess.check_call( shlex.split('ssh-keygen -f {} -R [localhost]:{}'.format( known_hosts, port))) id_pub = os.path.expanduser('~/.ssh/id_rsa.pub') if not os.path.isfile(id_pub): raise ClickableException( 'Could not find a ssh public key at "{}", please generate one and try again' .format(id_pub)) with open(id_pub, 'r') as f: public_key = f.read().strip() self.device.run_command('[ -d ~/.ssh ] || mkdir ~/.ssh', cwd=self.config.cwd) self.device.run_command('touch ~/.ssh/authorized_keys', cwd=self.config.cwd) output = run_subprocess_check_output( 'adb {} shell "grep \\"{}\\" ~/.ssh/authorized_keys"'.format( adb_args, public_key), shell=True).strip() if not output or 'No such file or directory' in output: logger.info('Inserting ssh public key on the connected device') self.device.run_command( 'echo \"{}\" >>~/.ssh/authorized_keys'.format(public_key), cwd=self.config.cwd) self.device.run_command('chmod 700 ~/.ssh', cwd=self.config.cwd) self.device.run_command('chmod 600 ~/.ssh/authorized_keys', cwd=self.config.cwd) subprocess.check_call( shlex.split( 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p {} phablet@localhost' .format(port))) self.toggle_ssh(on=False)
def run(self, path_arg=None): logger.info('Turning off device activity timeout') command = 'gsettings set com.ubuntu.touch.system activity-timeout 0' self.device.run_command(command, cwd=self.config.cwd)
def init_cmake_project(self, config, docker_config): executable = config.project_files.find_any_executable() exec_args = " ".join(config.project_files.find_any_exec_args()) #don't do all that if exec line not found if not executable: return choice = input( Colors.INFO + 'Do you want Clickable to setup a QtCreator project for you? [Y/n]: ' + Colors.CLEAR).strip().lower() if choice != 'y' and choice != 'yes' and choice != '': return #CLICK_EXE can be a variable match_exe_var = re.match("@([-\w]+)@", executable) if match_exe_var: #catch the variable name and try to get it from CMakeLists.txt cmd_var = match_exe_var.group(1) final_cmd = self.cmake_guess_exec_command(cmd_var) if final_cmd is not None: try: exe, exe_arg = final_cmd.split(' ', maxsplit=1) except: exe, exe_arg = final_cmd, '' executable = exe exec_args = exe_arg logger.debug( 'found that executable is {} with args: {}'.format( exe, exe_arg)) else: #was not able to guess executable logger.warning( "Could not determine executable command '{}', please adjust your project's run settings" .format(executable)) # work around for qtcreator bug when first run of a project to avoid qtcreator hang # we need to create the build directory first if not os.path.isdir(config.build_dir): os.makedirs(config.build_dir) env_vars = docker_config.environment clickable_env_path = '{}:{}'.format(env_vars["PATH"], env_vars["CLICK_PATH"]) clickable_ld_library_path = '{}:{}'.format( env_vars["LD_LIBRARY_PATH"], env_vars["CLICK_LD_LIBRARY_PATH"]) clickable_qml2_import_path = '{}:{}:{}'.format( env_vars["QML2_IMPORT_PATH"], env_vars["CLICK_QML2_IMPORT_PATH"], os.path.join(config.install_dir, 'lib')) template_replacement = { "CLICKABLE_LD_LIBRARY_PATH": clickable_ld_library_path, "CLICKABLE_QML2_IMPORT_PATH": clickable_qml2_import_path, "CLICKABLE_BUILD_DIR": config.build_dir, "CLICKABLE_INSTALL_DIR": config.install_dir, "CLICKABLE_EXEC_CMD": executable, "CLICKABLE_EXEC_ARGS": exec_args, "CLICKABLE_SRC_DIR": config.src_dir, "CLICKABLE_BUILD_ARGS": " ".join(config.build_args), "CLICKABLE_PATH": clickable_env_path } output_path = os.path.join(self.project_path, 'CMakeLists.txt.user.shared') #now read template and generate the .shared file to the root project dir with open(self.template_path, "r") as infile2, open(output_path, "w") as outfile: for line in infile2: for f_key, f_value in template_replacement.items(): if f_key in line: line = line.replace(f_key, f_value) outfile.write(line) logger.info( 'generated default build/run template to {}'.format(output_path))