def run(self, path_arg=None):
if not cookiecutter_available:
raise ClickableException(
'Cookiecutter is not available on your computer, more information can be found here: https://cookiecutter.readthedocs.io/en/latest/installation.html#install-cookiecutter'
)
config_file = os.path.expanduser(
'~/.clickable/cookiecutter_config.yaml')
if not os.path.isfile(config_file):
config_file = None
extra_context = {'Copyright Year': datetime.now().year}
if path_arg:
if path_arg in TEMPLATE_MAP:
extra_context['Template'] = TEMPLATE_MAP[path_arg]
else:
extra_context['Template'] = path_arg
no_input = not self.config.interactive
try:
cookiecutter.main.cookiecutter(
COOKIECUTTER_URL,
extra_context=extra_context,
no_input=no_input,
config_file=config_file,
)
except cookiecutter.exceptions.FailedHookException as err:
raise ClickableException('Failed to create app, see logs above')
def check_clickable_version(self):
if self.config['clickable_minimum_required']:
# Check if specified version string is valid
if not re.fullmatch("\d+(\.\d+)*",
self.config['clickable_minimum_required']):
raise ClickableException(
'"{}" specified as "clickable_minimum_required" is not a valid version number'
.format(self.config['clickable_minimum_required']))
# Convert version strings to integer lists
clickable_version_numbers = [
int(n) for n in re.split('\.', self.clickable_version)
]
clickable_required_numbers = [
int(n) for n in re.split(
'\.', self.config['clickable_minimum_required'])
]
if len(clickable_required_numbers) > len(
clickable_version_numbers):
logger.warning(
'Clickable version number only consists of {} numbers, but {} numbers specified in "clickable_minimum_required". Superfluous numbers will be ignored.'
.format(len(clickable_version_numbers),
len(clickable_required_numbers)))
# Compare all numbers until finding an unequal pair
for req, ver in zip(clickable_required_numbers,
clickable_version_numbers):
if req < ver:
break
if req > ver:
raise ClickableException(
'This project requires Clickable version {} ({} is used). Please update Clickable!'
.format(self.config['clickable_minimum_required'],
self.clickable_version))
def check_desktop_configs(self):
if self.debug_valgrind and self.debug_gdb:
raise ClickableException(
"Valgrind (--valgrind) and GDB (--gdb or --gdbserver) can not be combined."
)
if self.debug_valgrind and not self.is_desktop_mode():
raise ClickableException(
"Valgrind debugging is only supported in desktop mode! Consider running 'clickable desktop --valgrind'"
)
if self.debug_gdb and not self.is_desktop_mode():
raise ClickableException(
'"--gdb" and "--gdbserver" are flags for desktop mode. Use `clickable gdb` and `clickable gdbserver` for on-device debugging.'
)
if self.is_desktop_mode():
if self.use_nvidia and self.avoid_nvidia:
raise ClickableException(
'Configuration conflict: enforcing and avoiding nvidia mode must not be specified together.'
)
if self.container_mode:
raise ClickableException(
'Desktop Mode in Container Mode is not supported.')
def set_builder_interactive(self):
if self.config['builder'] or not self.needs_builder():
return
if not self.interactive:
raise ClickableException(
'No builder specified. Add a builder to your clickable.json.')
choice = input(
Colors.INFO +
'No builder was specified, would you like to auto detect the builder [y/N]: '
+ Colors.CLEAR).strip().lower()
if choice != 'y' and choice != 'yes':
raise ClickableException('Not auto detecting builder')
builder = None
directory = os.listdir(os.getcwd())
if 'config.xml' in directory:
builder = Constants.CORDOVA
if not builder and 'CMakeLists.txt' in directory:
builder = Constants.CMAKE
pro_files = [f for f in directory if f.endswith('.pro')]
if pro_files:
builder = Constants.QMAKE
if not builder:
builder = Constants.PURE
self.config['builder'] = builder
logger.info('Auto detected builder to be "{}"'.format(builder))
def find_binary_path(self):
desktop = self.config.install_files.get_desktop(
self.config.install_dir)
exec_list = desktop["Exec"].split()
binary = None
for arg in exec_list:
if "=" not in arg:
binary = arg
break
path = self.choose_executable(
[self.config.install_dir, self.config.app_bin_dir], binary)
if path:
if self.is_elf_file(path):
return path
else:
raise ClickableException(
'App executable "{}" is not an ELF file suitable for GDB debugging.'
.format(path))
if binary == "qmlscene":
raise ClickableException(
'Apps started via "qmlscene" are not supported by this debug method.'
)
else:
raise ClickableException(
'App binary "{}" found in desktop file could not be found in the app install directory. Please specify the path as "clickable gdb path/to/binary"'
.format(binary))
def determine_path_of_desktop_file(self, config):
desktop_path = None
hooks = config.get_manifest().get('hooks', {})
app = config.find_app_name()
if app:
if app in hooks and 'desktop' in hooks[app]:
desktop_path = hooks[app]['desktop']
else:
# TODO config.find_app_name never returns None. It raises an exception
for key, value in hooks.items():
if 'desktop' in value:
desktop_path = value['desktop']
break
if not desktop_path:
raise ClickableException('Could not find desktop file for app "{}"'.format(app))
# TODO finding the configured desktop entry should be moved to Config
# We could then proceed here with making it an absolute path and
# checking if it exists
desktop_path = os.path.join(config.install_dir, desktop_path)
if not os.path.exists(desktop_path):
raise ClickableException('Could not determine executable. Desktop file does not exist: "{}"'.format(desktop_path))
return desktop_path
def load_json_config(self, config_path):
config = {}
use_default_config = not config_path
if use_default_config:
config_path = os.path.join(self.cwd, 'clickable.json')
if os.path.isfile(config_path):
with open(config_path, 'r') as f:
config_json = {}
try:
config_json = json.load(f)
except ClickableException:
raise ClickableException(
'Failed reading "clickable.json", it is not valid json'
)
for key in self.removed_keywords:
if key in config_json:
raise ClickableException(
'"{}" is a no longer a valid configuration option'.
format(key))
schema = self.load_json_schema()
validate_clickable_json(config=config_json, schema=schema)
for key in self.config:
value = config_json.get(key, None)
if value:
config[key] = value
elif not use_default_config:
raise ClickableException(
'Specified config file {} does not exist.'.format(config_path))
return config
def run(self, path_arg=''):
if not requests_available:
raise ClickableException(
'Unable to publish app, python requests module is not installed'
)
if not self.config.apikey:
raise ClickableException(
'No api key specified, use OPENSTORE_API_KEY or --apikey')
click = self.config.install_files.get_click_filename()
click_path = os.path.join(self.config.build_dir, click)
url = OPENSTORE_API
if 'OPENSTORE_API' in os.environ and os.environ['OPENSTORE_API']:
url = os.environ['OPENSTORE_API']
package_name = self.config.install_files.find_package_name()
url = url + OPENSTORE_API_PATH.format(package_name)
channel = 'xenial'
files = {'file': open(click_path, 'rb')}
data = {
'channel': channel,
'changelog': path_arg.encode('utf8', 'surrogateescape'),
}
params = {'apikey': self.config.apikey}
logger.info(
'Uploading version {} of {} for {}/{} to the OpenStore'.format(
self.config.install_files.find_version(),
package_name,
channel,
self.config.arch,
))
response = requests.post(url, files=files, data=data, params=params)
if response.status_code == requests.codes.ok:
logger.info('Upload successful')
elif response.status_code == requests.codes.not_found:
title = urllib.parse.quote(
self.config.install_files.find_package_title())
raise ClickableException(
'App needs to be created in the OpenStore before you can publish it. Visit {}/submit?appId={}&name={}'
.format(
OPENSTORE_API,
package_name,
title,
))
else:
if response.text == 'Unauthorized':
raise ClickableException(
'Failed to upload click: Unauthorized')
else:
raise ClickableException('Failed to upload click: {}'.format(
response.json()['message']))
def check_arch_restrictions(self):
if self.is_arch_agnostic():
if self.config["arch"] != "all":
raise ClickableException(
'The "{}" builder needs architecture "all", but "{}" was specified'
.format(
self.config['builder'],
self.config['arch'],
))
if (self.config["restrict_arch"]
and self.config["restrict_arch"] != "all"):
raise ClickableException(
'The "{}" builder needs architecture "all", but "restrict_arch" was set to "{}"'
.format(
self.config['builder'],
self.config['restrict_arch'],
))
else:
if self.is_desktop_mode():
if self.config["arch"] != self.host_arch:
raise ClickableException(
'Desktop mode needs host architecture "{}", but "{}" was specified'
.format(self.host_arch, self.config["arch"]))
if (self.config['restrict_arch']
and self.config['restrict_arch'] != self.config['arch']):
raise ClickableException(
'Cannot build app for architecture "{}" as it is restricted to "{}" in the clickable.json.'
.format(self.config["arch"], self.config['restrict_arch']))
if (self.config['restrict_arch_env']
and self.config['restrict_arch_env'] != self.config['arch']
and self.config['arch'] != 'all' and self.is_build_cmd()):
raise ClickableException(
'Cannot build app for architecture "{}" as the environment is restricted to "{}".'
.format(self.config["arch"], self.config['restrict_arch_env']))
if self.config['arch'] == 'all':
install_keys = ['install_lib', 'install_bin', 'install_qml']
for key in install_keys:
if self.config[key]:
logger.warning(
"'{}' ({}) marked for install, even though architecture is 'all'."
.format("', '".join(self.config[key]), key))
if self.config['install_qml']:
logger.warning(
"Be aware that QML modules are going to be installed to {}, which is not part of 'QML2_IMPORT_PATH' at runtime."
.format(self.config['app_qml_dir']))
def check_cached_desktop_file(self):
path = self.get_cached_desktop_path()
try:
self.device.run_command("ls {} > /dev/null 2>&1".format(path),
get_output=True).strip()
except:
raise ClickableException('Failed to check installed version on device. The device is either not accessible or the app version you are trying to debug is not installed. Make sure the device is accessible or run "clickable install" and try again.')
def setup_image(self):
self.set_build_arch()
if self.needs_clickable_image():
self.check_nvidia_mode()
if self.use_nvidia and not self.build_arch.endswith('-nvidia'):
if self.is_ide_command():
self.build_arch = "{}-nvidia-ide".format(self.build_arch)
else:
self.build_arch = "{}-nvidia".format(self.build_arch)
if self.is_ide_command() and not self.use_nvidia:
self.build_arch = "{}-ide".format(self.build_arch)
image_framework = Constants.framework_image_mapping.get(
self.config['framework'], Constants.framework_fallback)
container_mapping_host = Constants.container_mapping[
self.host_arch]
if (image_framework,
self.build_arch) not in container_mapping_host:
raise ClickableException(
'There is currently no docker image for {}/{}'.format(
image_framework, self.build_arch))
self.config['docker_image'] = container_mapping_host[(
image_framework, self.build_arch)]
self.container_list = list(container_mapping_host.values())
def run(self, path_arg=""):
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Cleaning {}".format(lib.name))
found = True
if os.path.exists(lib.build_dir):
try:
shutil.rmtree(lib.build_dir)
except Exception:
cls, value, traceback = sys.exc_info()
if cls == OSError and 'No such file or directory' in str(
value
): # TODO see if there is a proper way to do this
pass # Nothing to do here, the directory didn't exist
else:
logger.warning(
'Failed to clean the build directory: {}: {}'.
format(type, value))
else:
logger.warning(
'Nothing to clean. Path does not exist: {}'.format(
lib.build_dir))
if single_lib and not found:
raise ClickableException(
'Cannot clean unknown library {}. You may add it to the clickable.json'
.format(single_lib))
def set_host_arch(self):
host = platform.machine()
self.host_arch = Constants.host_arch_mapping.get(host, None)
if not self.host_arch:
raise ClickableException(
"No support for host architecture {}".format(host))
def check_command(command):
error_code = run_subprocess_call(shlex.split('which {}'.format(command)),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if error_code != 0:
raise ClickableException(
'The command "{}" does not exist on this system, please install it for clickable to work properly"'
.format(command))
def build(self):
if os.path.isdir(self.config.install_dir):
raise ClickableException(
'Build directory already exists. Please run "clickable clean" before building again!'
)
shutil.copytree(self.config.cwd,
self.config.install_dir,
ignore=self._ignore)
logger.info('Copied files to install directory for click building')
def check_paths(self):
if self.is_build_cmd() and os.path.normpath(
self.cwd) == os.path.normpath(os.path.expanduser('~')):
raise ClickableException(
'Your are running a build command in your home directory.\nPlease navigate to an existing project or run "clickable create".'
)
if os.path.normpath(self.config['build_dir']) == os.path.normpath(
self.config['root_dir']):
raise ClickableException(
'Your "build_dir" is configured to be the same as your project "root_dir".\nPlease configure a sub-directory to avoid deleting your project on cleaning.'
)
if os.path.normpath(self.config['build_dir']) == os.path.normpath(
self.config['src_dir']):
raise ClickableException(
'Your "build_dir" is configured to be the same as your "src_dir".\nPlease configure different paths to avoid deleting your sources on cleaning.'
)
def load_manifest(self, manifest_path):
manifest = {}
with open(manifest_path, 'r') as f:
try:
manifest = json.load(f)
except ValueError:
raise ClickableException(
'Failed reading "manifest.json", it is not valid json')
return manifest
def find_package_name(self):
if self.builder == Constants.CORDOVA:
tree = ElementTree.parse('config.xml')
root = tree.getroot()
package = root.attrib['id'] if 'id' in root.attrib else None
if not package:
raise ClickableException(
'No package name specified in config.xml')
else:
package = self.get_manifest().get('name', None)
if not package:
raise ClickableException(
'No package name specified in manifest.json or clickable.json'
)
return package
def use_arch(self, build_arch):
if self.use_nvidia and not build_arch.endswith('-nvidia'):
build_arch = "{}-nvidia".format(build_arch)
if ('16.04', build_arch) not in self.container_mapping:
raise ClickableException(
'There is currently no docker image for 16.04/{}'.format(
build_arch))
self.config['docker_image'] = self.container_mapping[('16.04',
build_arch)]
def find_package_title(self):
if self.config['template'] == Config.CORDOVA:
tree = ElementTree.parse('config.xml')
root = tree.getroot()
title = root.attrib['name'] if 'name' in root.attrib else None
if not title:
raise ClickableException(
'No package title specified in config.xml')
else:
title = self.get_manifest().get('title', None)
if not title:
raise ClickableException(
'No package title specified in manifest.json or clickable.json'
)
return title
def get_make_jobs_from_args(make_args):
for arg in flexible_string_to_list(make_args):
if arg.startswith('-j'):
jobs_str = arg[2:]
try:
return int(jobs_str)
except ValueError:
raise ClickableException(
'"{}" in "make_args" is not a number, but it should be.')
return None
def load_json_schema(self):
schema_path = os.path.join(os.path.dirname(__file__),
'clickable.schema')
with open(schema_path, 'r') as f:
schema = {}
try:
return json.load(f)
except ClickableException:
raise ClickableException(
'Failed reading "clickable.schema", it is not valid json')
return None
def check_config_errors(self):
if not self.config['builder']:
raise ClickableException(
'The clickable.json is missing a "builder" in library "{}".'.
format(self.config["name"]))
if self.config[
'builder'] == Constants.CUSTOM and not self.config['build']:
raise ClickableException(
'When using the "custom" builder you must specify a "build" in one the lib configs'
)
if self.is_custom_docker_image:
if self.dependencies_host or self.dependencies_target or self.dependencies_ppa:
logger.warning(
"Dependencies are ignored when using a custom docker image!"
)
if self.image_setup:
logger.warning(
"Docker image setup is ignored when using a custom docker image!"
)
def set_arch(self, manifest):
arch = manifest.get('architecture', None)
if arch == '@CLICK_ARCH@' or arch == '':
manifest['architecture'] = self.config.arch
return True
if arch != self.config.arch:
raise ClickableException('Clickable is building for architecture "{}", but "{}" is specified in the manifest. You can set the architecture field to @CLICK_ARCH@ to let Clickable set the architecture field automatically.'.format(
self.config.arch, arch))
return False
def find_desktop_file(self):
desktop_path = None
hooks = self.config.install_files.get_manifest().get('hooks', {})
try:
app = self.config.install_files.find_app_name()
if app in hooks and 'desktop' in hooks[app]:
desktop_path = hooks[app]['desktop']
except ClickableException:
for key, value in hooks.items():
if 'desktop' in value:
desktop_path = value['desktop']
break
if not desktop_path:
raise ClickableException('Could not find desktop file for app')
desktop_path = os.path.join(self.config.install_dir, desktop_path)
if not os.path.exists(desktop_path):
raise ClickableException('Could not determine executable. Desktop file does not exist: "{}"'.format(desktop_path))
return desktop_path
def determine_executable(self, desktop_path):
execute = None
with open(desktop_path, 'r') as desktop_file:
for line in desktop_file.readlines():
if line.startswith('Exec='):
execute = line
break
if not execute:
raise ClickableException('No "Exec" line found in the desktop file {}'.format(desktop_path))
return execute[len('Exec='):].strip()
def run(self, path_arg=""):
if not self.config.lib_configs:
logger.warning('No libraries defined.')
single_lib = path_arg
found = False
for lib in self.config.lib_configs:
if not single_lib or single_lib == lib.name:
logger.info("Building {}".format(lib.name))
found = True
lib.container_mode = self.config.container_mode
lib.docker_image = self.config.docker_image
lib.build_arch = self.config.build_arch
lib.container = Container(lib, lib.name)
lib.container.setup()
# This is a workaround for lib env vars being overwritten by
# project env vars, especially affecting Container Mode.
lib.set_env_vars()
try:
os.makedirs(lib.build_dir, exist_ok=True)
except Exception:
logger.warning(
'Failed to create the build directory: {}'.format(
str(sys.exc_info()[0])))
try:
os.makedirs(lib.build_home, exist_ok=True)
except Exception:
logger.warning(
'Failed to create the build home directory: {}'.format(
str(sys.exc_info()[0])))
if lib.prebuild:
run_subprocess_check_call(lib.prebuild,
cwd=self.config.cwd,
shell=True)
self.build(lib)
if lib.postbuild:
run_subprocess_check_call(lib.postbuild,
cwd=lib.build_dir,
shell=True)
if single_lib and not found:
raise ClickableException(
'Cannot build unknown library {}, which is not in your clickable.json'
.format(single_lib))
def install_files(self, pattern, dest_dir):
if not is_sub_dir(dest_dir, self.config.install_dir):
dest_dir = os.path.abspath(self.config.install_dir + "/" + dest_dir)
makedirs(dest_dir)
if '"' in pattern:
# Make sure one cannot run random bash code through the "ls" command
raise ClickableException("install_* patterns must not contain any '\"' quotation character.")
command = 'ls -d "{}"'.format(pattern)
files = self.config.container.run_command(command, get_output=True).split()
logger.info("Installing {}".format(", ".join(files)))
self.config.container.pull_files(files, dest_dir)
def check_docker(self, retries=3):
if not self.docker_mode:
raise ClickableException(
"Container was not initialized with Container Mode. This seems to be a bug in Clickable."
)
if self.needs_docker_setup():
self.setup_docker()
try:
run_subprocess_check_output(shlex.split('docker ps'),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
retries -= 1
if retries <= 0:
raise ClickableException(
"Couldn't check docker. If you just installed Clickable you may need to reboot once."
)
self.start_docker()
time.sleep(3) # Give it a sec to boot up
self.check_docker(retries)
def check_builder_rules(self):
if not self.needs_builder():
return
if self.config[
'builder'] == Constants.CUSTOM and not self.config['build']:
raise ClickableException(
'When using the "custom" builder you must specify a "build" in the config'
)
if self.config['builder'] == Constants.GO and not self.config['gopath']:
raise ClickableException(
'When using the "go" builder you must specify a "gopath" in the config or use the '
'"GOPATH" env variable')
if self.config[
'builder'] == Constants.RUST and not self.config['cargo_home']:
raise ClickableException(
'When using the "rust" builder you must specify a "cargo_home" in the config'
)
if self.config['builder'] and self.config[
'builder'] not in Constants.builders:
raise ClickableException('"{}" is not a valid builder ({})'.format(
self.config['builder'], ', '.join(Constants.builders)))