def run(self, path_arg=None):
try:
os.makedirs(self.config.build_dir)
except FileExistsError:
pass
except Exception:
logger.warning('Failed to create the build directory: {}'.format(
str(sys.exc_info()[0])))
self.config.container.setup_dependencies()
if self.config.prebuild:
run_subprocess_check_call(self.config.prebuild,
cwd=self.config.cwd,
shell=True)
self.build()
self.install_additional_files()
if self.config.postbuild:
run_subprocess_check_call(self.config.postbuild,
cwd=self.config.build_dir,
shell=True)
self.click_build()
def run(self, path_arg=None):
if os.path.exists(self.config.build_dir):
try:
shutil.rmtree(self.config.build_dir)
except Exception:
cls, value, traceback = sys.exc_info()
if cls == OSError and 'No such file or directory' in str(
value): # TODO see if there is a proper way to do this
pass # Nothing to do here, the directory didn't exist
else:
logger.warning(
'Failed to clean the build directory: {}: {}'.format(
type, value))
if os.path.exists(self.config.install_dir):
try:
shutil.rmtree(self.config.install_dir)
except Exception:
cls, value, traceback = sys.exc_info()
if cls == OSError and 'No such file or directory' in str(
value): # TODO see if there is a proper way to do this
pass # Nothing to do here, the directory didn't exist
else:
logger.warning(
'Failed to clean the temp directory: {}: {}'.format(
type, value))
def run(self, path_arg=None):
try:
os.makedirs(self.config.build_dir, exist_ok=True)
except Exception:
logger.warning('Failed to create the build directory: {}'.format(str(sys.exc_info()[0])))
try:
os.makedirs(self.config.build_home, exist_ok=True)
except Exception:
logger.warning('Failed to create the build home directory: {}'.format(str(sys.exc_info()[0])))
self.config.container.setup()
if self.config.prebuild:
run_subprocess_check_call(self.config.prebuild, cwd=self.config.cwd, shell=True)
self.build()
self.install_additional_files()
if self.config.postbuild:
run_subprocess_check_call(self.config.postbuild, cwd=self.config.build_dir, shell=True)
self.click_build()
if not self.config.skip_review:
review = ReviewCommand(self.config)
review.check(self.click_path, raise_on_error=False)
def run(self, path_arg=""):
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Cleaning {}".format(lib.name))
found = True
if os.path.exists(lib.build_dir):
try:
shutil.rmtree(lib.build_dir)
except Exception:
cls, value, traceback = sys.exc_info()
if cls == OSError and 'No such file or directory' in str(
value
): # TODO see if there is a proper way to do this
pass # Nothing to do here, the directory didn't exist
else:
logger.warning(
'Failed to clean the build directory: {}: {}'.
format(type, value))
else:
logger.warning(
'Nothing to clean. Path does not exist: {}'.format(
lib.build_dir))
if single_lib and not found:
raise ClickableException(
'Cannot clean unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def run(self, path_arg=None):
devices = self.device.detect_attached()
if len(devices) == 0:
logger.warning('No attached devices')
else:
for device in devices:
logger.info(device)
def setup_docker(self):
self.config.container.check_docker()
if is_command('xhost'):
subprocess.check_call(shlex.split('xhost +local:docker'))
else:
logger.warning('xhost not installed, desktop mode may fail')
return self.setup_docker_config()
def make_install(self):
if os.path.exists(self.config.install_dir) and os.path.isdir(self.config.install_dir):
shutil.rmtree(self.config.install_dir)
try:
os.makedirs(self.config.install_dir)
except FileExistsError:
logger.warning('Failed to create temp dir, already exists')
except Exception:
logger.warning('Failed to create temp dir ({}): {}'.format(self.config.install_dir, str(sys.exc_info()[0])))
def set_framework(self, manifest):
framework = manifest.get('framework', None)
if framework == '@CLICK_FRAMEWORK@' or framework == '':
manifest['framework'] = self.config.framework
return True
if framework != self.config.framework:
logger.warning('Framework in manifest is "{}", Clickable expected "{}".'.format(
framework, self.config.framework))
return False
def cleanup_config(self):
if not self.config['make_jobs']:
self.config['make_jobs'] = multiprocessing.cpu_count()
self.make_args = merge_make_jobs_into_args(make_args=self.make_args,
make_jobs=self.make_jobs)
if self.config['dependencies_build']:
self.config['dependencies_host'] += self.config[
'dependencies_build']
self.config['dependencies_build'] = []
logger.warning(
'"dependencies_build" is deprecated. Use "dependencies_host" instead!'
)
for key in self.flexible_lists:
self.config[key] = flexible_string_to_list(self.config[key])
def start_gdbserver(self):
if not self.config.ssh:
logger.warning('SSH is recommended for the "gdbserver" command. If you experience any issues, try again with "--ssh"')
app_exec = self.get_app_exec_full_path()
desktop_file = self.get_cached_desktop_path()
app_dir = self.get_app_dir()
environ = self.get_app_env()
set_env = " ".join(["{}='{}'".format(key, value) for key, value in environ.items()])
commands = [
'cd {}'.format(app_dir),
'{} gdbserver localhost:{} {} --desktop_file_hint={}'.format(
set_env, self.port, app_exec, desktop_file)
]
self.set_signal_handler()
self.device.run_command(commands, forward_port=self.port)
def run_test(self, lib):
if not os.path.exists(lib.build_dir):
logger.warning(
"Library {} has not yet been built for host architecture.".
format(lib.name))
else:
lib.container_mode = self.config.container_mode
lib.docker_image = self.config.docker_image
lib.build_arch = self.config.build_arch
lib.container = Container(lib, lib.name)
lib.container.setup()
# This is a workaround for lib env vars being overwritten by
# project env vars, especially affecting Container Mode.
lib.set_env_vars()
command = 'xvfb-startup {}'.format(lib.test)
lib.container.run_command(command, use_build_dir=True)
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Running tests on {}".format(lib.name))
found = True
self.run_test(lib)
if single_lib and not found:
raise ClickableException(
'Cannot test unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Building {}".format(lib.name))
found = True
lib.container_mode = self.config.container_mode
lib.docker_image = self.config.docker_image
lib.build_arch = self.config.build_arch
lib.container = Container(lib, lib.name)
lib.container.setup_dependencies()
try:
os.makedirs(lib.build_dir)
except FileExistsError:
pass
except Exception:
logger.warning(
'Failed to create the build directory: {}'.format(
str(sys.exc_info()[0])))
if lib.prebuild:
run_subprocess_check_call(lib.prebuild,
cwd=self.config.cwd,
shell=True)
self.build(lib)
if lib.postbuild:
run_subprocess_check_call(lib.postbuild,
cwd=lib.build_dir,
shell=True)
if single_lib and not found:
raise ClickableException(
'Cannot build unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def restore_cached_container(self):
with open(self.docker_name_file, 'r') as f:
cached_container = f.read().strip()
if not image_exists(cached_container):
logger.warning("Cached container does not exist anymore")
return
command_base = 'docker images -q {}'.format(self.base_docker_image)
command_cached = 'docker history -q {}'.format(cached_container)
hash_base = run_subprocess_check_output(command_base).strip()
history_cached = run_subprocess_check_output(
command_cached).strip()
if hash_base in history_cached:
logger.debug("Found cached container")
self.docker_image = cached_container
else:
logger.warning("Found outdated container")
def check_config_errors(self):
if not self.config['builder']:
raise ClickableException(
'The clickable.json is missing a "builder" in library "{}".'.
format(self.config["name"]))
if self.config[
'builder'] == Constants.CUSTOM and not self.config['build']:
raise ClickableException(
'When using the "custom" builder you must specify a "build" in one the lib configs'
)
if self.is_custom_docker_image:
if self.dependencies_host or self.dependencies_target or self.dependencies_ppa:
logger.warning(
"Dependencies are ignored when using a custom docker image!"
)
if self.image_setup:
logger.warning(
"Docker image setup is ignored when using a custom docker image!"
)
def validate_clickable_json(config, schema):
try:
from jsonschema import validate, ValidationError
try:
validate(instance=config, schema=schema)
except ValidationError as e:
logger.error("The clickable.json configuration file is invalid!")
error_message = e.message
# Lets add the key to the invalid value
if e.path:
if len(e.path) > 1 and isinstance(e.path[-1], int):
error_message = "{} (in '{}')".format(
error_message, e.path[-2])
else:
error_message = "{} (in '{}')".format(
error_message, e.path[-1])
raise ClickableException(error_message)
except ImportError:
logger.warning(
"Dependency 'jsonschema' not found. Could not validate clickable.json."
)
pass
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Building {}".format(lib.name))
found = True
lib.container_mode = self.config.container_mode
lib.docker_image = self.config.docker_image
lib.build_arch = self.config.build_arch
lib.container = Container(lib, lib.name)
lib.container.setup()
# This is a workaround for lib env vars being overwritten by
# project env vars, especially affecting Container Mode.
lib.set_env_vars()
try:
os.makedirs(lib.build_dir, exist_ok=True)
except Exception:
logger.warning(
'Failed to create the build directory: {}'.format(
str(sys.exc_info()[0])))
try:
os.makedirs(lib.build_home, exist_ok=True)
except Exception:
logger.warning(
'Failed to create the build home directory: {}'.format(
str(sys.exc_info()[0])))
if lib.prebuild:
run_subprocess_check_call(lib.prebuild,
cwd=self.config.cwd,
shell=True)
self.build(lib)
if lib.postbuild:
run_subprocess_check_call(lib.postbuild,
cwd=lib.build_dir,
shell=True)
if single_lib and not found:
raise ClickableException(
'Cannot build unknown library {}, which is not in your clickable.json'
.format(single_lib))
def restore_cached_image(self):
if not os.path.exists(self.docker_name_file):
return
with open(self.docker_name_file, 'r') as f:
cached_image = None
cached_base_image = None
try:
image_file = json.load(f)
cached_image = image_file.get('name', None)
cached_base_image = image_file.get('base_image', None)
except ValueError:
pass
if not cached_image:
logger.warning("Cached image file is invalid")
return
if not image_exists(cached_image):
logger.warning("Cached container does not exist anymore")
return
if self.base_docker_image != cached_base_image:
logger.warning("Cached image has a different base image")
self.check_docker()
command_base = 'docker images -q {}'.format(self.base_docker_image)
command_cached = 'docker history -q {}'.format(cached_image)
hash_base = run_subprocess_check_output(command_base).strip()
history_cached = run_subprocess_check_output(
command_cached).strip()
if hash_base in history_cached:
logger.debug("Found cached container")
self.docker_image = cached_image
else:
logger.warning("Cached container is outdated")
def check_version(self, quiet=False):
if requests_available:
version = None
check = True
version_check = expanduser('~/.clickable/version_check.json')
if isfile(version_check):
with open(version_check, 'r') as f:
try:
version_check_data = json.load(f)
except ValueError:
version_check_data = None
if version_check_data and 'version' in version_check_data and 'datetime' in version_check_data:
last_check = datetime.strptime(
version_check_data['datetime'], DATE_FORMAT)
if last_check > (datetime.now() - timedelta(days=2)) and \
'current_version' in version_check_data and \
version_check_data['current_version'] == __version__:
check = False
version = version_check_data['version']
logger.debug('Using cached version check')
if check:
logger.debug('Checking for updates to clickable')
try:
response = requests.get(
'https://clickable-ut.dev/en/latest/_static/version.json',
timeout=5)
response.raise_for_status()
data = response.json()
version = data['version']
except requests.exceptions.Timeout as e:
logger.warning(
'Unable to check for updates to clickable, the request timedout'
)
except Exception as e:
logger.debug('Version check failed:' + str(e.cmd),
exc_info=e)
logger.warning(
'Unable to check for updates to clickable, an unknown error occurred'
)
if version:
with open(version_check, 'w') as f:
json.dump(
{
'version': version,
'datetime':
datetime.now().strftime(DATE_FORMAT),
'current_version': __version__,
}, f)
if version:
if version != __version__:
logger.info(
'v{} of clickable is available, update to get the latest features and improvements!'
.format(version))
else:
if not quiet:
logger.info(
'You are running the latest version of clickable!')
else:
if not quiet:
logger.warning(
'Unable to check for updates to clickable, please install "requests"'
)
def setup_dependencies(self, force_build=False):
if self.config.dependencies_build or self.config.dependencies_target:
logger.debug('Checking dependencies')
dependencies = self.config.dependencies_build
for dep in self.config.dependencies_target:
if ':' in dep:
dependencies.append(dep)
else:
dependencies.append('{}:{}'.format(dep, self.config.arch))
if self.config.container_mode:
self.run_command('apt-get update', sudo=True, use_dir=False)
command = 'apt-get install -y --force-yes'
run = False
for dep in dependencies:
exists = ''
try:
exists = self.run_command(
'dpkg -s {} | grep Status'.format(dep),
get_output=True,
use_dir=False)
except subprocess.CalledProcessError:
exists = ''
if exists.strip() != 'Status: install ok installed':
run = True
command = '{} {}'.format(command, dep)
if run:
self.run_command(command, sudo=True, use_dir=False)
else:
logger.debug('Dependencies already installed')
else:
self.check_docker()
if self.config.custom_docker_image:
logger.warning(
'Skipping dependency check, using a custom docker image'
)
else:
command_ppa = ''
if self.config.dependencies_ppa:
command_ppa = 'RUN add-apt-repository {}'.format(
' '.join(self.config.dependencies_ppa))
dockerfile = '''
FROM {}
RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate
{}
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean
'''.format(self.base_docker_image, command_ppa,
' '.join(dependencies)).strip()
build = force_build
if not os.path.exists(self.clickable_dir):
os.makedirs(self.clickable_dir)
if self.docker_image != self.base_docker_image and os.path.exists(
self.docker_file):
with open(self.docker_file, 'r') as f:
if dockerfile.strip() != f.read().strip():
build = True
else:
build = True
if not build:
command = 'docker images -q {}'.format(
self.docker_image)
image_exists = run_subprocess_check_output(
command).strip()
build = not image_exists
if build:
with open(self.docker_file, 'w') as f:
f.write(dockerfile)
self.docker_image = '{}-{}'.format(
self.base_docker_image, uuid.uuid4())
with open(self.docker_name_file, 'w') as f:
f.write(self.docker_image)
logger.debug('Generating new docker image')
try:
subprocess.check_call(shlex.split(
'docker build -t {} .'.format(
self.docker_image)),
cwd=self.clickable_dir)
except subprocess.CalledProcessError:
self.clean_clickable()
raise
else:
logger.debug('Dependencies already setup')
def __init__(self, name, json_config, arch, root_dir, qt_version,
debug_build, verbose):
# Must come after ARCH_TRIPLET to avoid breaking it
self.placeholders.update({"ARCH": "arch"})
self.qt_version = qt_version
self.debug_build = debug_build
self.verbose = verbose
self.set_host_arch()
self.container_list = list(
Constants.container_mapping[self.host_arch].values())
self.config = {
'name': name,
'arch': arch,
'arch_triplet': None,
'template': None,
'builder': None,
'postmake': None,
'prebuild': None,
'build': None,
'postbuild': None,
'build_dir': '${ROOT}/build/${ARCH_TRIPLET}/${NAME}',
'build_home': '${BUILD_DIR}/.clickable/home',
'src_dir': '${ROOT}/libs/${NAME}',
'root_dir': root_dir,
'dependencies_build': [],
'dependencies_host': [],
'dependencies_target': [],
'dependencies_ppa': [],
'make_jobs': None,
'docker_image': None,
'build_args': [],
'env_vars': {},
'make_args': [],
'install_dir': '${BUILD_DIR}/install',
'image_setup': {},
'test': 'ctest',
}
# TODO remove support for deprecated "template" in clickable.json
if "template" in json_config:
logger.warning(
'Parameter "template" is deprecated in clickable.json. Use "builder" as drop-in replacement instead.'
)
json_config["builder"] = json_config["template"]
json_config["template"] = None
self.config.update(json_config)
if self.config["docker_image"]:
self.is_custom_docker_image = True
else:
self.is_custom_docker_image = False
self.cleanup_config()
self.config['arch_triplet'] = Constants.arch_triplet_mapping[
self.config['arch']]
for key in self.path_keys:
if key not in self.accepts_placeholders and self.config[key]:
self.config[key] = os.path.abspath(self.config[key])
self.substitute_placeholders()
self.set_env_vars()
self.check_config_errors()
for key, value in self.config.items():
logger.debug('Lib {} config value {}: {}'.format(name, key, value))
def allow_docker_to_connect_to_xserver(self):
if self.is_xhost_installed():
subprocess.check_call(shlex.split('xhost +local:docker'))
else:
logger.warning('xhost not installed, desktop mode may fail')
def init_cmake_project(self, config, docker_config):
executable = config.project_files.find_any_executable()
exec_args = " ".join(config.project_files.find_any_exec_args())
#don't do all that if exec line not found
if not executable:
return
choice = input(
Colors.INFO +
'Do you want Clickable to setup a QtCreator project for you? [Y/n]: '
+ Colors.CLEAR).strip().lower()
if choice != 'y' and choice != 'yes' and choice != '':
return
#CLICK_EXE can be a variable
match_exe_var = re.match("@([-\w]+)@", executable)
if match_exe_var:
#catch the variable name and try to get it from CMakeLists.txt
cmd_var = match_exe_var.group(1)
final_cmd = self.cmake_guess_exec_command(cmd_var)
if final_cmd is not None:
try:
exe, exe_arg = final_cmd.split(' ', maxsplit=1)
except:
exe, exe_arg = final_cmd, ''
executable = exe
exec_args = exe_arg
logger.debug(
'found that executable is {} with args: {}'.format(
exe, exe_arg))
else:
#was not able to guess executable
logger.warning(
"Could not determine executable command '{}', please adjust your project's run settings"
.format(executable))
# work around for qtcreator bug when first run of a project to avoid qtcreator hang
# we need to create the build directory first
if not os.path.isdir(config.build_dir):
os.makedirs(config.build_dir)
env_vars = docker_config.environment
clickable_env_path = '{}:{}'.format(env_vars["PATH"],
env_vars["CLICK_PATH"])
clickable_ld_library_path = '{}:{}'.format(
env_vars["LD_LIBRARY_PATH"], env_vars["CLICK_LD_LIBRARY_PATH"])
clickable_qml2_import_path = '{}:{}:{}'.format(
env_vars["QML2_IMPORT_PATH"], env_vars["CLICK_QML2_IMPORT_PATH"],
os.path.join(config.install_dir, 'lib'))
template_replacement = {
"CLICKABLE_LD_LIBRARY_PATH": clickable_ld_library_path,
"CLICKABLE_QML2_IMPORT_PATH": clickable_qml2_import_path,
"CLICKABLE_BUILD_DIR": config.build_dir,
"CLICKABLE_INSTALL_DIR": config.install_dir,
"CLICKABLE_EXEC_CMD": executable,
"CLICKABLE_EXEC_ARGS": exec_args,
"CLICKABLE_SRC_DIR": config.src_dir,
"CLICKABLE_BUILD_ARGS": " ".join(config.build_args),
"CLICKABLE_PATH": clickable_env_path
}
output_path = os.path.join(self.project_path,
'CMakeLists.txt.user.shared')
#now read template and generate the .shared file to the root project dir
with open(self.template_path, "r") as infile2, open(output_path,
"w") as outfile:
for line in infile2:
for f_key, f_value in template_replacement.items():
if f_key in line:
line = line.replace(f_key, f_value)
outfile.write(line)
logger.info(
'generated default build/run template to {}'.format(output_path))
def run(self, path_arg=None):
logger.warning(
'The click-build command has been merged into the build command. Please remove this command from your CI, as this warning will be removed in a future version.'
)
def run(self, path_arg=None):
try:
self.config.container.run_command("echo ''", use_build_dir=False)
logger.info('Clickable is set up and ready.')
except ClickableException:
logger.warning('Please log out or restart to apply changes')
def print_instructions(self):
instructions = self.query.get_user_instructions()
if instructions is not None:
logger.warning(instructions)
def run(self, arg_commands=[], args=None):
self.config = self.setup_config(args, arg_commands)
self.config.container = Container(
self.config, minimum_version=__container_minimum_required__)
commands = self.config.commands
VALID_COMMANDS = self.command_names + list(self.config.scripts.keys())
is_default = not arg_commands
'''
Detect senarios when an argument is passed to a command. For example:
`clickable install /path/to/click`. Since clickable allows commands
to be strung together it makes detecting this harder. This check has
been limited to just the case when we have 2 values in args.commands as
stringing together multiple commands and a command with an argument is
unlikely to occur.
TODO remove chaining and clean this up
'''
command_arg = ''
if len(commands) == 2 and commands[1] not in VALID_COMMANDS:
command_arg = commands[1]
commands = commands[:1]
commands = [
self.command_aliases[command]
if command in self.command_aliases else command
for command in commands
]
if len(commands) > 1 and not is_default:
logger.warning(
'Chaining multiple commands is deprecated and will be rejected in a future version of Clickable.'
)
for command in commands:
if command in self.command_names:
cmd = self.command_classes[command](self.config)
cmd.preprocess(command_arg)
for command in commands:
if command == 'bash-completion':
cli_args = [
'--serial-number',
'--config',
'--ssh',
'--arch',
'--verbose',
'--container-mode',
'--apikey',
'--docker-image',
'--dirty',
'--debug',
]
print(' '.join(sorted(VALID_COMMANDS + cli_args)))
elif command == 'bash-completion-desktop':
cli_args = [
'--nvidia',
'--no-nvidia'
'--gdbserver',
'--gdb',
'--dark-mode',
'--lang',
'--skip-build',
'--dirty',
'--verbose',
'--config',
]
print(' '.join(sorted(cli_args)))
elif command in self.config.scripts:
logger.debug('Running the "{}" script'.format(command))
subprocess.check_call(self.config.scripts[command],
cwd=self.config.cwd,
shell=True)
elif command in self.command_names:
logger.debug('Running the "{}" command'.format(command))
cmd = self.command_classes[command](self.config)
cmd.run(command_arg)
else:
logger.error(
'There is no builtin or custom command named "{}"'.format(
command))
self.print_valid_commands()
sys.exit(1)
