def run(self, path_arg=None): try: os.makedirs(self.config.build_dir) except FileExistsError: pass except Exception: logger.warning('Failed to create the build directory: {}'.format( str(sys.exc_info()[0]))) self.config.container.setup_dependencies() if self.config.prebuild: run_subprocess_check_call(self.config.prebuild, cwd=self.config.cwd, shell=True) self.build() self.install_additional_files() if self.config.postbuild: run_subprocess_check_call(self.config.postbuild, cwd=self.config.build_dir, shell=True) self.click_build()
def run(self, path_arg=None): if os.path.exists(self.config.build_dir): try: shutil.rmtree(self.config.build_dir) except Exception: cls, value, traceback = sys.exc_info() if cls == OSError and 'No such file or directory' in str( value): # TODO see if there is a proper way to do this pass # Nothing to do here, the directory didn't exist else: logger.warning( 'Failed to clean the build directory: {}: {}'.format( type, value)) if os.path.exists(self.config.install_dir): try: shutil.rmtree(self.config.install_dir) except Exception: cls, value, traceback = sys.exc_info() if cls == OSError and 'No such file or directory' in str( value): # TODO see if there is a proper way to do this pass # Nothing to do here, the directory didn't exist else: logger.warning( 'Failed to clean the temp directory: {}: {}'.format( type, value))
def run(self, path_arg=None): try: os.makedirs(self.config.build_dir, exist_ok=True) except Exception: logger.warning('Failed to create the build directory: {}'.format(str(sys.exc_info()[0]))) try: os.makedirs(self.config.build_home, exist_ok=True) except Exception: logger.warning('Failed to create the build home directory: {}'.format(str(sys.exc_info()[0]))) self.config.container.setup() if self.config.prebuild: run_subprocess_check_call(self.config.prebuild, cwd=self.config.cwd, shell=True) self.build() self.install_additional_files() if self.config.postbuild: run_subprocess_check_call(self.config.postbuild, cwd=self.config.build_dir, shell=True) self.click_build() if not self.config.skip_review: review = ReviewCommand(self.config) review.check(self.click_path, raise_on_error=False)
def run(self, path_arg=""): single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Cleaning {}".format(lib.name)) found = True if os.path.exists(lib.build_dir): try: shutil.rmtree(lib.build_dir) except Exception: cls, value, traceback = sys.exc_info() if cls == OSError and 'No such file or directory' in str( value ): # TODO see if there is a proper way to do this pass # Nothing to do here, the directory didn't exist else: logger.warning( 'Failed to clean the build directory: {}: {}'. format(type, value)) else: logger.warning( 'Nothing to clean. Path does not exist: {}'.format( lib.build_dir)) if single_lib and not found: raise ClickableException( 'Cannot clean unknown library {}. You may add it to the clickable.json' .format(single_lib))
def run(self, path_arg=None): devices = self.device.detect_attached() if len(devices) == 0: logger.warning('No attached devices') else: for device in devices: logger.info(device)
def setup_docker(self): self.config.container.check_docker() if is_command('xhost'): subprocess.check_call(shlex.split('xhost +local:docker')) else: logger.warning('xhost not installed, desktop mode may fail') return self.setup_docker_config()
def make_install(self): if os.path.exists(self.config.install_dir) and os.path.isdir(self.config.install_dir): shutil.rmtree(self.config.install_dir) try: os.makedirs(self.config.install_dir) except FileExistsError: logger.warning('Failed to create temp dir, already exists') except Exception: logger.warning('Failed to create temp dir ({}): {}'.format(self.config.install_dir, str(sys.exc_info()[0])))
def set_framework(self, manifest): framework = manifest.get('framework', None) if framework == '@CLICK_FRAMEWORK@' or framework == '': manifest['framework'] = self.config.framework return True if framework != self.config.framework: logger.warning('Framework in manifest is "{}", Clickable expected "{}".'.format( framework, self.config.framework)) return False
def cleanup_config(self): if not self.config['make_jobs']: self.config['make_jobs'] = multiprocessing.cpu_count() self.make_args = merge_make_jobs_into_args(make_args=self.make_args, make_jobs=self.make_jobs) if self.config['dependencies_build']: self.config['dependencies_host'] += self.config[ 'dependencies_build'] self.config['dependencies_build'] = [] logger.warning( '"dependencies_build" is deprecated. Use "dependencies_host" instead!' ) for key in self.flexible_lists: self.config[key] = flexible_string_to_list(self.config[key])
def start_gdbserver(self): if not self.config.ssh: logger.warning('SSH is recommended for the "gdbserver" command. If you experience any issues, try again with "--ssh"') app_exec = self.get_app_exec_full_path() desktop_file = self.get_cached_desktop_path() app_dir = self.get_app_dir() environ = self.get_app_env() set_env = " ".join(["{}='{}'".format(key, value) for key, value in environ.items()]) commands = [ 'cd {}'.format(app_dir), '{} gdbserver localhost:{} {} --desktop_file_hint={}'.format( set_env, self.port, app_exec, desktop_file) ] self.set_signal_handler() self.device.run_command(commands, forward_port=self.port)
def run_test(self, lib): if not os.path.exists(lib.build_dir): logger.warning( "Library {} has not yet been built for host architecture.". format(lib.name)) else: lib.container_mode = self.config.container_mode lib.docker_image = self.config.docker_image lib.build_arch = self.config.build_arch lib.container = Container(lib, lib.name) lib.container.setup() # This is a workaround for lib env vars being overwritten by # project env vars, especially affecting Container Mode. lib.set_env_vars() command = 'xvfb-startup {}'.format(lib.test) lib.container.run_command(command, use_build_dir=True)
def run(self, path_arg=""): if not self.config.lib_configs: logger.warning('No libraries defined.') single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Running tests on {}".format(lib.name)) found = True self.run_test(lib) if single_lib and not found: raise ClickableException( 'Cannot test unknown library {}. You may add it to the clickable.json' .format(single_lib))
def run(self, path_arg=""): if not self.config.lib_configs: logger.warning('No libraries defined.') single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Building {}".format(lib.name)) found = True lib.container_mode = self.config.container_mode lib.docker_image = self.config.docker_image lib.build_arch = self.config.build_arch lib.container = Container(lib, lib.name) lib.container.setup_dependencies() try: os.makedirs(lib.build_dir) except FileExistsError: pass except Exception: logger.warning( 'Failed to create the build directory: {}'.format( str(sys.exc_info()[0]))) if lib.prebuild: run_subprocess_check_call(lib.prebuild, cwd=self.config.cwd, shell=True) self.build(lib) if lib.postbuild: run_subprocess_check_call(lib.postbuild, cwd=lib.build_dir, shell=True) if single_lib and not found: raise ClickableException( 'Cannot build unknown library {}. You may add it to the clickable.json' .format(single_lib))
def restore_cached_container(self): with open(self.docker_name_file, 'r') as f: cached_container = f.read().strip() if not image_exists(cached_container): logger.warning("Cached container does not exist anymore") return command_base = 'docker images -q {}'.format(self.base_docker_image) command_cached = 'docker history -q {}'.format(cached_container) hash_base = run_subprocess_check_output(command_base).strip() history_cached = run_subprocess_check_output( command_cached).strip() if hash_base in history_cached: logger.debug("Found cached container") self.docker_image = cached_container else: logger.warning("Found outdated container")
def check_config_errors(self): if not self.config['builder']: raise ClickableException( 'The clickable.json is missing a "builder" in library "{}".'. format(self.config["name"])) if self.config[ 'builder'] == Constants.CUSTOM and not self.config['build']: raise ClickableException( 'When using the "custom" builder you must specify a "build" in one the lib configs' ) if self.is_custom_docker_image: if self.dependencies_host or self.dependencies_target or self.dependencies_ppa: logger.warning( "Dependencies are ignored when using a custom docker image!" ) if self.image_setup: logger.warning( "Docker image setup is ignored when using a custom docker image!" )
def validate_clickable_json(config, schema): try: from jsonschema import validate, ValidationError try: validate(instance=config, schema=schema) except ValidationError as e: logger.error("The clickable.json configuration file is invalid!") error_message = e.message # Lets add the key to the invalid value if e.path: if len(e.path) > 1 and isinstance(e.path[-1], int): error_message = "{} (in '{}')".format( error_message, e.path[-2]) else: error_message = "{} (in '{}')".format( error_message, e.path[-1]) raise ClickableException(error_message) except ImportError: logger.warning( "Dependency 'jsonschema' not found. Could not validate clickable.json." ) pass
def run(self, path_arg=""): if not self.config.lib_configs: logger.warning('No libraries defined.') single_lib = path_arg found = False for lib in self.config.lib_configs: if not single_lib or single_lib == lib.name: logger.info("Building {}".format(lib.name)) found = True lib.container_mode = self.config.container_mode lib.docker_image = self.config.docker_image lib.build_arch = self.config.build_arch lib.container = Container(lib, lib.name) lib.container.setup() # This is a workaround for lib env vars being overwritten by # project env vars, especially affecting Container Mode. lib.set_env_vars() try: os.makedirs(lib.build_dir, exist_ok=True) except Exception: logger.warning( 'Failed to create the build directory: {}'.format( str(sys.exc_info()[0]))) try: os.makedirs(lib.build_home, exist_ok=True) except Exception: logger.warning( 'Failed to create the build home directory: {}'.format( str(sys.exc_info()[0]))) if lib.prebuild: run_subprocess_check_call(lib.prebuild, cwd=self.config.cwd, shell=True) self.build(lib) if lib.postbuild: run_subprocess_check_call(lib.postbuild, cwd=lib.build_dir, shell=True) if single_lib and not found: raise ClickableException( 'Cannot build unknown library {}, which is not in your clickable.json' .format(single_lib))
def restore_cached_image(self): if not os.path.exists(self.docker_name_file): return with open(self.docker_name_file, 'r') as f: cached_image = None cached_base_image = None try: image_file = json.load(f) cached_image = image_file.get('name', None) cached_base_image = image_file.get('base_image', None) except ValueError: pass if not cached_image: logger.warning("Cached image file is invalid") return if not image_exists(cached_image): logger.warning("Cached container does not exist anymore") return if self.base_docker_image != cached_base_image: logger.warning("Cached image has a different base image") self.check_docker() command_base = 'docker images -q {}'.format(self.base_docker_image) command_cached = 'docker history -q {}'.format(cached_image) hash_base = run_subprocess_check_output(command_base).strip() history_cached = run_subprocess_check_output( command_cached).strip() if hash_base in history_cached: logger.debug("Found cached container") self.docker_image = cached_image else: logger.warning("Cached container is outdated")
def check_version(self, quiet=False): if requests_available: version = None check = True version_check = expanduser('~/.clickable/version_check.json') if isfile(version_check): with open(version_check, 'r') as f: try: version_check_data = json.load(f) except ValueError: version_check_data = None if version_check_data and 'version' in version_check_data and 'datetime' in version_check_data: last_check = datetime.strptime( version_check_data['datetime'], DATE_FORMAT) if last_check > (datetime.now() - timedelta(days=2)) and \ 'current_version' in version_check_data and \ version_check_data['current_version'] == __version__: check = False version = version_check_data['version'] logger.debug('Using cached version check') if check: logger.debug('Checking for updates to clickable') try: response = requests.get( 'https://clickable-ut.dev/en/latest/_static/version.json', timeout=5) response.raise_for_status() data = response.json() version = data['version'] except requests.exceptions.Timeout as e: logger.warning( 'Unable to check for updates to clickable, the request timedout' ) except Exception as e: logger.debug('Version check failed:' + str(e.cmd), exc_info=e) logger.warning( 'Unable to check for updates to clickable, an unknown error occurred' ) if version: with open(version_check, 'w') as f: json.dump( { 'version': version, 'datetime': datetime.now().strftime(DATE_FORMAT), 'current_version': __version__, }, f) if version: if version != __version__: logger.info( 'v{} of clickable is available, update to get the latest features and improvements!' .format(version)) else: if not quiet: logger.info( 'You are running the latest version of clickable!') else: if not quiet: logger.warning( 'Unable to check for updates to clickable, please install "requests"' )
def setup_dependencies(self, force_build=False): if self.config.dependencies_build or self.config.dependencies_target: logger.debug('Checking dependencies') dependencies = self.config.dependencies_build for dep in self.config.dependencies_target: if ':' in dep: dependencies.append(dep) else: dependencies.append('{}:{}'.format(dep, self.config.arch)) if self.config.container_mode: self.run_command('apt-get update', sudo=True, use_dir=False) command = 'apt-get install -y --force-yes' run = False for dep in dependencies: exists = '' try: exists = self.run_command( 'dpkg -s {} | grep Status'.format(dep), get_output=True, use_dir=False) except subprocess.CalledProcessError: exists = '' if exists.strip() != 'Status: install ok installed': run = True command = '{} {}'.format(command, dep) if run: self.run_command(command, sudo=True, use_dir=False) else: logger.debug('Dependencies already installed') else: self.check_docker() if self.config.custom_docker_image: logger.warning( 'Skipping dependency check, using a custom docker image' ) else: command_ppa = '' if self.config.dependencies_ppa: command_ppa = 'RUN add-apt-repository {}'.format( ' '.join(self.config.dependencies_ppa)) dockerfile = ''' FROM {} RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate {} RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean '''.format(self.base_docker_image, command_ppa, ' '.join(dependencies)).strip() build = force_build if not os.path.exists(self.clickable_dir): os.makedirs(self.clickable_dir) if self.docker_image != self.base_docker_image and os.path.exists( self.docker_file): with open(self.docker_file, 'r') as f: if dockerfile.strip() != f.read().strip(): build = True else: build = True if not build: command = 'docker images -q {}'.format( self.docker_image) image_exists = run_subprocess_check_output( command).strip() build = not image_exists if build: with open(self.docker_file, 'w') as f: f.write(dockerfile) self.docker_image = '{}-{}'.format( self.base_docker_image, uuid.uuid4()) with open(self.docker_name_file, 'w') as f: f.write(self.docker_image) logger.debug('Generating new docker image') try: subprocess.check_call(shlex.split( 'docker build -t {} .'.format( self.docker_image)), cwd=self.clickable_dir) except subprocess.CalledProcessError: self.clean_clickable() raise else: logger.debug('Dependencies already setup')
def __init__(self, name, json_config, arch, root_dir, qt_version, debug_build, verbose): # Must come after ARCH_TRIPLET to avoid breaking it self.placeholders.update({"ARCH": "arch"}) self.qt_version = qt_version self.debug_build = debug_build self.verbose = verbose self.set_host_arch() self.container_list = list( Constants.container_mapping[self.host_arch].values()) self.config = { 'name': name, 'arch': arch, 'arch_triplet': None, 'template': None, 'builder': None, 'postmake': None, 'prebuild': None, 'build': None, 'postbuild': None, 'build_dir': '${ROOT}/build/${ARCH_TRIPLET}/${NAME}', 'build_home': '${BUILD_DIR}/.clickable/home', 'src_dir': '${ROOT}/libs/${NAME}', 'root_dir': root_dir, 'dependencies_build': [], 'dependencies_host': [], 'dependencies_target': [], 'dependencies_ppa': [], 'make_jobs': None, 'docker_image': None, 'build_args': [], 'env_vars': {}, 'make_args': [], 'install_dir': '${BUILD_DIR}/install', 'image_setup': {}, 'test': 'ctest', } # TODO remove support for deprecated "template" in clickable.json if "template" in json_config: logger.warning( 'Parameter "template" is deprecated in clickable.json. Use "builder" as drop-in replacement instead.' ) json_config["builder"] = json_config["template"] json_config["template"] = None self.config.update(json_config) if self.config["docker_image"]: self.is_custom_docker_image = True else: self.is_custom_docker_image = False self.cleanup_config() self.config['arch_triplet'] = Constants.arch_triplet_mapping[ self.config['arch']] for key in self.path_keys: if key not in self.accepts_placeholders and self.config[key]: self.config[key] = os.path.abspath(self.config[key]) self.substitute_placeholders() self.set_env_vars() self.check_config_errors() for key, value in self.config.items(): logger.debug('Lib {} config value {}: {}'.format(name, key, value))
def allow_docker_to_connect_to_xserver(self): if self.is_xhost_installed(): subprocess.check_call(shlex.split('xhost +local:docker')) else: logger.warning('xhost not installed, desktop mode may fail')
def init_cmake_project(self, config, docker_config): executable = config.project_files.find_any_executable() exec_args = " ".join(config.project_files.find_any_exec_args()) #don't do all that if exec line not found if not executable: return choice = input( Colors.INFO + 'Do you want Clickable to setup a QtCreator project for you? [Y/n]: ' + Colors.CLEAR).strip().lower() if choice != 'y' and choice != 'yes' and choice != '': return #CLICK_EXE can be a variable match_exe_var = re.match("@([-\w]+)@", executable) if match_exe_var: #catch the variable name and try to get it from CMakeLists.txt cmd_var = match_exe_var.group(1) final_cmd = self.cmake_guess_exec_command(cmd_var) if final_cmd is not None: try: exe, exe_arg = final_cmd.split(' ', maxsplit=1) except: exe, exe_arg = final_cmd, '' executable = exe exec_args = exe_arg logger.debug( 'found that executable is {} with args: {}'.format( exe, exe_arg)) else: #was not able to guess executable logger.warning( "Could not determine executable command '{}', please adjust your project's run settings" .format(executable)) # work around for qtcreator bug when first run of a project to avoid qtcreator hang # we need to create the build directory first if not os.path.isdir(config.build_dir): os.makedirs(config.build_dir) env_vars = docker_config.environment clickable_env_path = '{}:{}'.format(env_vars["PATH"], env_vars["CLICK_PATH"]) clickable_ld_library_path = '{}:{}'.format( env_vars["LD_LIBRARY_PATH"], env_vars["CLICK_LD_LIBRARY_PATH"]) clickable_qml2_import_path = '{}:{}:{}'.format( env_vars["QML2_IMPORT_PATH"], env_vars["CLICK_QML2_IMPORT_PATH"], os.path.join(config.install_dir, 'lib')) template_replacement = { "CLICKABLE_LD_LIBRARY_PATH": clickable_ld_library_path, "CLICKABLE_QML2_IMPORT_PATH": clickable_qml2_import_path, "CLICKABLE_BUILD_DIR": config.build_dir, "CLICKABLE_INSTALL_DIR": config.install_dir, "CLICKABLE_EXEC_CMD": executable, "CLICKABLE_EXEC_ARGS": exec_args, "CLICKABLE_SRC_DIR": config.src_dir, "CLICKABLE_BUILD_ARGS": " ".join(config.build_args), "CLICKABLE_PATH": clickable_env_path } output_path = os.path.join(self.project_path, 'CMakeLists.txt.user.shared') #now read template and generate the .shared file to the root project dir with open(self.template_path, "r") as infile2, open(output_path, "w") as outfile: for line in infile2: for f_key, f_value in template_replacement.items(): if f_key in line: line = line.replace(f_key, f_value) outfile.write(line) logger.info( 'generated default build/run template to {}'.format(output_path))
def run(self, path_arg=None): logger.warning( 'The click-build command has been merged into the build command. Please remove this command from your CI, as this warning will be removed in a future version.' )
def run(self, path_arg=None): try: self.config.container.run_command("echo ''", use_build_dir=False) logger.info('Clickable is set up and ready.') except ClickableException: logger.warning('Please log out or restart to apply changes')
def print_instructions(self): instructions = self.query.get_user_instructions() if instructions is not None: logger.warning(instructions)
def run(self, arg_commands=[], args=None): self.config = self.setup_config(args, arg_commands) self.config.container = Container( self.config, minimum_version=__container_minimum_required__) commands = self.config.commands VALID_COMMANDS = self.command_names + list(self.config.scripts.keys()) is_default = not arg_commands ''' Detect senarios when an argument is passed to a command. For example: `clickable install /path/to/click`. Since clickable allows commands to be strung together it makes detecting this harder. This check has been limited to just the case when we have 2 values in args.commands as stringing together multiple commands and a command with an argument is unlikely to occur. TODO remove chaining and clean this up ''' command_arg = '' if len(commands) == 2 and commands[1] not in VALID_COMMANDS: command_arg = commands[1] commands = commands[:1] commands = [ self.command_aliases[command] if command in self.command_aliases else command for command in commands ] if len(commands) > 1 and not is_default: logger.warning( 'Chaining multiple commands is deprecated and will be rejected in a future version of Clickable.' ) for command in commands: if command in self.command_names: cmd = self.command_classes[command](self.config) cmd.preprocess(command_arg) for command in commands: if command == 'bash-completion': cli_args = [ '--serial-number', '--config', '--ssh', '--arch', '--verbose', '--container-mode', '--apikey', '--docker-image', '--dirty', '--debug', ] print(' '.join(sorted(VALID_COMMANDS + cli_args))) elif command == 'bash-completion-desktop': cli_args = [ '--nvidia', '--no-nvidia' '--gdbserver', '--gdb', '--dark-mode', '--lang', '--skip-build', '--dirty', '--verbose', '--config', ] print(' '.join(sorted(cli_args))) elif command in self.config.scripts: logger.debug('Running the "{}" script'.format(command)) subprocess.check_call(self.config.scripts[command], cwd=self.config.cwd, shell=True) elif command in self.command_names: logger.debug('Running the "{}" command'.format(command)) cmd = self.command_classes[command](self.config) cmd.run(command_arg) else: logger.error( 'There is no builtin or custom command named "{}"'.format( command)) self.print_valid_commands() sys.exit(1)