def get_time_zone(self):
try:
return run_subprocess_check_output(
'timedatectl show -p Timezone --value',
stderr=subprocess.DEVNULL)
except:
logger.debug(
'timedatectl show command failed. Falling back to alternative way to detect timezone...'
)
if os.path.exists('/etc/timezone'):
with open('/etc/timezone') as host_timezone_file:
return host_timezone_file.readline().strip()
else:
logger.debug(
'/etc/timezone does not exist. Falling back to alternative way to detect timezone...'
)
try:
output = run_subprocess_check_output('timedatectl status')
for line in output.splitlines():
line = line.strip()
if line.startswith('Time zone:'):
start = line.find(':') + 1
end = line.find('(')
return line[start:end].strip()
except:
logger.debug(
"timedatctl status method failed to set timezone from host in desktop mode..."
)
logger.debug("Falling back to UTC as timezone.")
return 'UTC'
def run(self, path_arg=None):
'''
Inspired by http://bazaar.launchpad.net/~phablet-team/phablet-tools/trunk/view/head:/phablet-shell
'''
if self.config.ssh:
subprocess.check_call(shlex.split('ssh phablet@{}'.format(self.config.ssh)))
else:
self.device.check_any_attached()
adb_args = ''
if self.config.device_serial_number:
adb_args = '-s {}'.format(self.config.device_serial_number)
else:
self.device.check_multiple_attached()
output = run_subprocess_check_output(shlex.split('adb {} shell pgrep sshd'.format(adb_args))).split()
if not output:
self.toggle_ssh(on=True)
# Use the usb cable rather than worrying about going over wifi
port = 0
for p in range(2222, 2299):
error_code = run_subprocess_call(shlex.split('adb {} forward tcp:{} tcp:22'.format(adb_args, p)), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if error_code == 0:
port = p
break
if port == 0:
raise Exception('Failed to open a port to the device')
# Purge the device host key so that SSH doesn't print a scary warning about it
# (it changes every time the device is reflashed and this is expected)
known_hosts = os.path.expanduser('~/.ssh/known_hosts')
subprocess.check_call(shlex.split('touch {}'.format(known_hosts)))
subprocess.check_call(shlex.split('ssh-keygen -f {} -R [localhost]:{}'.format(known_hosts, port)))
id_pub = os.path.expanduser('~/.ssh/id_rsa.pub')
if not os.path.isfile(id_pub):
raise Exception('Could not find a ssh public key at "{}", please generate one and try again'.format(id_pub))
with open(id_pub, 'r') as f:
public_key = f.read().strip()
self.device.run_command('[ -d ~/.ssh ] || mkdir ~/.ssh', cwd=self.config.cwd)
self.device.run_command('touch ~/.ssh/authorized_keys', cwd=self.config.cwd)
output = run_subprocess_check_output('adb {} shell "grep \\"{}\\" ~/.ssh/authorized_keys"'.format(adb_args, public_key), shell=True).strip()
if not output or 'No such file or directory' in output:
print_info('Inserting ssh public key on the connected device')
self.device.run_command('echo \"{}\" >>~/.ssh/authorized_keys'.format(public_key), cwd=self.config.cwd)
self.device.run_command('chmod 700 ~/.ssh', cwd=self.config.cwd)
self.device.run_command('chmod 600 ~/.ssh/authorized_keys', cwd=self.config.cwd)
subprocess.check_call(shlex.split('ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p {} phablet@localhost'.format(port)))
self.toggle_ssh(on=False)
def check_docker(self, retries=3):
try:
run_subprocess_check_output(shlex.split('docker ps'),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
retries -= 1
if retries <= 0:
raise e
self.start_docker()
time.sleep(3) # Give it a sec to boot up
self.check_docker(retries)
def check_docker(self, retries=3):
if self.needs_setup():
self.setup_docker()
try:
run_subprocess_check_output(shlex.split('docker ps'), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
retries -= 1
if retries <= 0:
raise e
self.start_docker()
time.sleep(3) # Give it a sec to boot up
self.check_docker(retries)
def pull_files(self, files, dst_parent):
os.makedirs(dst_parent, exist_ok=True)
if self.config.container_mode:
for f in files:
dst_path = os.path.join(dst_parent, os.path.basename(f))
if os.path.isdir(f):
if os.path.exists(dst_path):
shutil.rmtree(dst_path)
shutil.copytree(f, dst_path)
else:
if os.path.exists(dst_path):
os.remove(dst_path)
shutil.copy(f, dst_parent, follow_symlinks=False)
else: # Docker
mounts = self.render_mounts(
self.get_docker_mounts(transparent=[self.config.root_dir]))
command_create = 'docker create {} {}'.format(
mounts, self.docker_image)
container = run_subprocess_check_output(command_create).strip()
for f in files:
command_copy = 'docker cp {}:{} {}'.format(
container, f, dst_parent)
run_subprocess_check_call(command_copy)
command_remove = 'docker rm {}'.format(container)
run_subprocess_check_call(command_remove,
stdout=subprocess.DEVNULL)
def setup_docker(self):
self.check_command('docker')
self.start_docker()
group_exists = False
with open('/etc/group', 'r') as f:
lines = f.readlines()
for line in lines:
if line.startswith('docker:'):
group_exists = True
if not group_exists:
print_info('Asking for root to create docker group')
subprocess.check_call(shlex.split('sudo groupadd docker'))
output = run_subprocess_check_output(
shlex.split('groups {}'.format(getpass.getuser()))).strip()
# Test for exactly docker in the group list
if ' docker ' in output or output.endswith(
' docker') or output.startswith(
'docker ') or output == 'docker':
print_info('Setup has already been completed')
else:
print_info(
'Asking for root to add the current user to the docker group')
subprocess.check_call(
shlex.split('sudo usermod -aG docker {}'.format(
getpass.getuser())))
print_info('Log out or restart to apply changes')
def check_lxd(self):
name = 'clickable-{}'.format(self.config.build_arch)
status = ''
try:
status = run_subprocess_check_output(shlex.split('usdk-target status {}'.format(name)), stderr=subprocess.STDOUT)
status = json.loads(status)['status']
except subprocess.CalledProcessError as e:
if e.output.strip() == 'error: Could not connect to the LXD server.' or 'Can\'t establish a working socket connection' in e.output.strip():
started = self.start_lxd()
if started:
status = 'Running' # Pretend it's started, but we will call this function again to check if it's actually ok
time.sleep(3) # Give it a sec to boot up
self.check_lxd()
else:
raise Exception('LXD is not running, please start it')
elif e.output.strip() == 'error: Could not query container status. error: not found':
raise Exception('No lxd container exists to build in, please run `clickable setup-lxd`')
else:
print(e.output)
raise e
if status != 'Running':
print_info('Going to start lxd container "{}"'.format(name))
subprocess.check_call(shlex.split('lxc start {}'.format(name)))
def user_part_of_docker_group(self):
output = run_subprocess_check_output(
shlex.split('groups {}'.format(getpass.getuser()))).strip()
# Test for exactly docker in the group list
return (' docker ' in output or output.endswith(' docker')
or output.startswith('docker ') or output == 'docker')
def check_lxd(self):
name = 'clickable-{}'.format(self.config.build_arch)
status = ''
try:
status = run_subprocess_check_output(shlex.split(
'usdk-target status {}'.format(name)),
stderr=subprocess.STDOUT)
status = json.loads(status)['status']
except subprocess.CalledProcessError as e:
if e.output.strip(
) == 'error: Could not connect to the LXD server.' or 'Can\'t establish a working socket connection' in e.output.strip(
):
started = self.start_lxd()
if started:
status = 'Running' # Pretend it's started, but we will call this function again to check if it's actually ok
time.sleep(3) # Give it a sec to boot up
self.check_lxd()
else:
raise Exception('LXD is not running, please start it')
elif e.output.strip(
) == 'error: Could not query container status. error: not found':
raise Exception(
'No lxd container exists to build in, please run `clickable setup-lxd`'
)
else:
print(e.output)
raise e
if status != 'Running':
print_info('Going to start lxd container "{}"'.format(name))
subprocess.check_call(shlex.split('lxc start {}'.format(name)))
def run_container_command(self,
command,
force_lxd=False,
sudo=False,
get_output=False,
use_dir=True,
cwd=None):
wrapped_command = command
cwd = cwd if cwd else self.cwd
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
elif force_lxd or self.config.lxd:
self.check_lxd()
target_command = 'exec'
if sudo:
target_command = 'maint'
if use_dir:
command = 'cd {}; {}'.format(self.config.dir, command)
wrapped_command = 'usdk-target {} clickable-{} -- bash -c "{}"'.format(
target_command, self.build_arch, command)
elif self.config.chroot:
chroot_command = 'run'
if sudo:
chroot_command = 'maint'
wrapped_command = 'click chroot -a {} -f {} {} bash -c "{}"'.format(
self.build_arch, self.config.sdk, chroot_command, command)
else: # Docker
self.check_docker()
go_config = ''
if self.config.gopath:
go_config = '-v {}:/gopath -e GOPATH=/gopath'.format(
self.config.gopath)
wrapped_command = 'docker run -v {}:{} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format(
cwd,
cwd,
go_config,
self.config.dir if use_dir else cwd,
os.getuid(),
self.docker_image,
command,
)
kwargs = {}
if use_dir:
kwargs['cwd'] = self.config.dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command),
**kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def is_met(self):
if not is_program_installed('nvidia-smi'):
return False
modules = run_subprocess_check_output('lsmod').splitlines()
for m in modules:
if m.split(' ', 1)[0] == 'nvidia':
return True
return False
def restore_cached_container(self):
with open(self.docker_name_file, 'r') as f:
cached_container = f.read().strip()
if not image_exists(cached_container):
logger.warning("Cached container does not exist anymore")
return
command_base = 'docker images -q {}'.format(self.base_docker_image)
command_cached = 'docker history -q {}'.format(cached_container)
hash_base = run_subprocess_check_output(command_base).strip()
history_cached = run_subprocess_check_output(
command_cached).strip()
if hash_base in history_cached:
logger.debug("Found cached container")
self.docker_image = cached_container
else:
logger.warning("Found outdated container")
def restore_cached_image(self):
if not os.path.exists(self.docker_name_file):
return
with open(self.docker_name_file, 'r') as f:
cached_image = None
cached_base_image = None
try:
image_file = json.load(f)
cached_image = image_file.get('name', None)
cached_base_image = image_file.get('base_image', None)
except ValueError:
pass
if not cached_image:
logger.warning("Cached image file is invalid")
return
if not image_exists(cached_image):
logger.warning("Cached container does not exist anymore")
return
if self.base_docker_image != cached_base_image:
logger.warning("Cached image has a different base image")
self.check_docker()
command_base = 'docker images -q {}'.format(self.base_docker_image)
command_cached = 'docker history -q {}'.format(cached_image)
hash_base = run_subprocess_check_output(command_base).strip()
history_cached = run_subprocess_check_output(
command_cached).strip()
if hash_base in history_cached:
logger.debug("Found cached container")
self.docker_image = cached_image
else:
logger.warning("Cached container is outdated")
def lxd_container_exists(self):
name = 'clickable-{}'.format(self.config.build_arch)
# Check for existing container
existing = run_subprocess_check_output(shlex.split('{} list'.format(self.usdk_target)))
existing = json.loads(existing)
found = False
for container in existing:
if container['name'] == name:
found = True
return found
def check_docker(self, retries=3):
if not self.docker_mode:
raise ClickableException(
"Container was not initialized with Container Mode. This seems to be a bug in Clickable."
)
if self.needs_docker_setup():
self.setup_docker()
try:
run_subprocess_check_output(shlex.split('docker ps'),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
retries -= 1
if retries <= 0:
raise ClickableException(
"Couldn't check docker. If you just installed Clickable you may need to reboot once."
)
self.start_docker()
time.sleep(3) # Give it a sec to boot up
self.check_docker(retries)
def run(self, path_arg=None):
self.container.check_docker()
command = 'docker pull {}'.format(self.container.base_docker_image)
run_subprocess_check_call(command)
if 'armhf' in self.container.base_docker_image:
image = self.container.base_docker_image.replace('armhf', 'amd64')
command = 'docker images -q {}'.format(image)
image_exists = run_subprocess_check_output(command).strip()
if image_exists:
command = 'docker pull {}'.format(image)
run_subprocess_check_call(command)
def lxd_container_exists(self):
name = 'clickable-{}'.format(self.config.build_arch)
# Check for existing container
existing = run_subprocess_check_output(
shlex.split('{} list'.format(self.usdk_target)))
existing = json.loads(existing)
found = False
for container in existing:
if container['name'] == name:
found = True
return found
def is_dockerfile_outdated(self, dockerfile_content):
if self.docker_image == self.base_docker_image:
return True
if not os.path.exists(self.clickable_dir):
return True
if not os.path.exists(self.docker_file):
return True
with open(self.docker_file, 'r') as f:
if dockerfile_content.strip() != f.read().strip():
return True
command = 'docker images -q {}'.format(self.docker_image)
image_exists = run_subprocess_check_output(command).strip()
return not image_exists
def detect_devices(self):
output = run_subprocess_check_output(
shlex.split('adb devices -l')).strip()
devices = []
for line in output.split('\n'):
if 'device' in line and 'devices' not in line:
device = line.split(' ')[0]
for part in line.split(' '):
if part.startswith('model:'):
device = '{} - {}'.format(
device,
part.replace('model:', '').replace('_',
' ').strip())
devices.append(device)
return devices
def check_base_image_version(self):
if not self.minimum_version:
return
if not image_exists(self.docker_image):
return
version = 0
try:
format_string = '{{ index .Config.Labels "image_version"}}'
command = "docker inspect --format '{}' {}".format(
format_string, self.docker_image)
logger.debug(
'Checking docker image version via: {}'.format(command))
version_string = run_subprocess_check_output(command)
version = int(version_string)
except (ValueError, subprocess.CalledProcessError):
logger.warn("Could not read the image version from the container")
if version < self.minimum_version:
raise ClickableException(
'This version of Clickable requires Clickable docker image {} in version {} or higher (found version {}). Please run "clickable update" to update your local images.'
.format(self.docker_image, self.minimum_version, version))
def run_command(self,
command,
force_lxd=False,
sudo=False,
get_output=False,
use_dir=True,
cwd=None):
wrapped_command = command
cwd = cwd if cwd else self.config.cwd
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
elif force_lxd or self.config.lxd:
self.check_lxd()
target_command = 'exec'
if sudo:
target_command = 'maint'
if use_dir:
command = 'cd {}; {}'.format(self.config.dir, command)
wrapped_command = 'usdk-target {} clickable-{} -- bash -c "{}"'.format(
target_command, self.config.build_arch, command)
else: # Docker
self.check_docker()
if ' ' in cwd or ' ' in self.config.dir:
raise Exception(
'There are spaces in the current path, this will cause errors in the build process'
)
if self.config.first_docker_info:
print_info('Using docker container "{}"'.format(
self.docker_image))
self.config.first_docker_info = False
go_config = ''
if self.config.gopath:
go_config = '-v {}:/gopath -e GOPATH=/gopath'.format(
self.config.gopath)
wrapped_command = 'docker run -v {}:{} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format(
cwd,
cwd,
go_config,
self.config.dir if use_dir else cwd,
os.getuid(),
self.docker_image,
command,
)
kwargs = {}
if use_dir:
kwargs['cwd'] = self.config.dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command),
**kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def run_command(self, command, force_lxd=False, sudo=False, get_output=False, use_dir=True, cwd=None):
wrapped_command = command
cwd = cwd if cwd else os.path.abspath(self.config.root_dir)
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
elif force_lxd or self.config.lxd:
self.check_lxd()
target_command = 'exec'
if sudo:
target_command = 'maint'
if use_dir:
command = 'cd {}; {}'.format(self.config.build_dir, command)
wrapped_command = 'usdk-target {} clickable-{} -- bash -c "{}"'.format(target_command, self.config.build_arch, command)
else: # Docker
self.check_docker()
if ' ' in cwd or ' ' in self.config.build_dir:
raise Exception('There are spaces in the current path, this will cause errors in the build process')
if self.config.first_docker_info:
print_info('Using docker container "{}"'.format(self.docker_image))
self.config.first_docker_info = False
go_config = ''
if self.config.gopath:
gopaths = self.config.gopath.split(':')
docker_gopaths = []
go_configs = []
for (index, path) in enumerate(gopaths):
go_configs.append('-v {}:/gopath/path{}:Z'.format(path, index))
docker_gopaths.append('/gopath/path{}'.format(index))
go_config = '{} -e GOPATH={}'.format(
' '.join(go_configs),
':'.join(docker_gopaths),
)
rust_config = ''
if self.config.config['template'] == Config.RUST and self.config.cargo_home:
cargo_registry = os.path.join(self.config.cargo_home, 'registry')
cargo_git = os.path.join(self.config.cargo_home, 'git')
os.makedirs(cargo_registry, exist_ok=True)
os.makedirs(cargo_git, exist_ok=True)
rust_config = '-v {}:/opt/rust/cargo/registry:Z -v {}:/opt/rust/cargo/git:Z'.format(
cargo_registry,
cargo_git,
)
wrapped_command = 'docker run -v {}:{}:Z {} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format(
cwd,
cwd,
go_config,
rust_config,
self.config.build_dir if use_dir else cwd,
os.getuid(),
self.docker_image,
command,
)
kwargs = {}
if use_dir:
kwargs['cwd'] = self.config.build_dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def setup_dependencies(self, force_build=False):
if self.config.dependencies_build or self.config.dependencies_target:
logger.debug('Checking dependencies')
dependencies = self.config.dependencies_build
for dep in self.config.dependencies_target:
if ':' in dep:
dependencies.append(dep)
else:
dependencies.append('{}:{}'.format(dep, self.config.arch))
if self.config.container_mode:
self.run_command('apt-get update', sudo=True, use_dir=False)
command = 'apt-get install -y --force-yes'
run = False
for dep in dependencies:
exists = ''
try:
exists = self.run_command(
'dpkg -s {} | grep Status'.format(dep),
get_output=True,
use_dir=False)
except subprocess.CalledProcessError:
exists = ''
if exists.strip() != 'Status: install ok installed':
run = True
command = '{} {}'.format(command, dep)
if run:
self.run_command(command, sudo=True, use_dir=False)
else:
logger.debug('Dependencies already installed')
else:
self.check_docker()
if self.config.custom_docker_image:
logger.warning(
'Skipping dependency check, using a custom docker image'
)
else:
command_ppa = ''
if self.config.dependencies_ppa:
command_ppa = 'RUN add-apt-repository {}'.format(
' '.join(self.config.dependencies_ppa))
dockerfile = '''
FROM {}
RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate
{}
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean
'''.format(self.base_docker_image, command_ppa,
' '.join(dependencies)).strip()
build = force_build
if not os.path.exists(self.clickable_dir):
os.makedirs(self.clickable_dir)
if self.docker_image != self.base_docker_image and os.path.exists(
self.docker_file):
with open(self.docker_file, 'r') as f:
if dockerfile.strip() != f.read().strip():
build = True
else:
build = True
if not build:
command = 'docker images -q {}'.format(
self.docker_image)
image_exists = run_subprocess_check_output(
command).strip()
build = not image_exists
if build:
with open(self.docker_file, 'w') as f:
f.write(dockerfile)
self.docker_image = '{}-{}'.format(
self.base_docker_image, uuid.uuid4())
with open(self.docker_name_file, 'w') as f:
f.write(self.docker_image)
logger.debug('Generating new docker image')
try:
subprocess.check_call(shlex.split(
'docker build -t {} .'.format(
self.docker_image)),
cwd=self.clickable_dir)
except subprocess.CalledProcessError:
self.clean_clickable()
raise
else:
logger.debug('Dependencies already setup')
def run_command(self,
command,
sudo=False,
get_output=False,
use_dir=True,
cwd=None):
wrapped_command = command
cwd = cwd if cwd else os.path.abspath(self.config.root_dir)
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
else: # Docker
self.check_docker()
if ' ' in cwd or ' ' in self.config.build_dir:
raise ClickableException(
'There are spaces in the current path, this will cause errors in the build process'
)
if self.config.first_docker_info:
logger.debug('Using docker container "{}"'.format(
self.docker_image))
self.config.first_docker_info = False
go_config = ''
if self.config.gopath:
gopaths = self.config.gopath.split(':')
docker_gopaths = []
go_configs = []
for (index, path) in enumerate(gopaths):
go_configs.append('-v {}:/gopath/path{}:Z'.format(
path, index))
docker_gopaths.append('/gopath/path{}'.format(index))
go_config = '{} -e GOPATH={}'.format(
' '.join(go_configs),
':'.join(docker_gopaths),
)
rust_config = ''
if self.config.config[
'template'] == Config.RUST and self.config.cargo_home:
logger.info("Caching cargo related files in {}".format(
self.config.cargo_home))
cargo_registry = os.path.join(self.config.cargo_home,
'registry')
cargo_git = os.path.join(self.config.cargo_home, 'git')
cargo_package_cache_lock = os.path.join(
self.config.cargo_home, '.package-cache')
os.makedirs(cargo_registry, exist_ok=True)
os.makedirs(cargo_git, exist_ok=True)
# create .package-cache if it doesn't exist
with open(cargo_package_cache_lock, "a"):
pass
rust_config = '-v {}:/opt/rust/cargo/registry:Z -v {}:/opt/rust/cargo/git:Z -v {}:/opt/rust/cargo/.package-cache'.format(
cargo_registry,
cargo_git,
cargo_package_cache_lock,
)
env_vars = self.config.prepare_docker_env_vars()
wrapped_command = 'docker run -v {}:{}:Z {} {} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format(
cwd,
cwd,
env_vars,
go_config,
rust_config,
self.config.build_dir if use_dir else cwd,
os.getuid(),
self.docker_image,
command,
)
kwargs = {}
if use_dir:
kwargs['cwd'] = self.config.build_dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command),
**kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def setup_dependencies(self):
if self.config.dependencies_build or self.config.dependencies_target:
print_info('Checking dependencies')
dependencies = self.config.dependencies_build
for dep in self.config.dependencies_target:
if ':' in dep:
dependencies.append(dep)
else:
dependencies.append('{}:{}'.format(dep, self.config.arch))
if self.config.lxd or self.config.container_mode:
self.run_command('apt-get update', sudo=True, use_dir=False)
command = 'apt-get install -y --force-yes'
run = False
for dep in dependencies:
exists = ''
try:
exists = self.run_command('dpkg -s {} | grep Status'.format(dep), get_output=True, use_dir=False)
except subprocess.CalledProcessError:
exists = ''
if exists.strip() != 'Status: install ok installed':
run = True
command = '{} {}'.format(command, dep)
if run:
self.run_command(command, sudo=True, use_dir=False)
else:
print_info('Dependencies already installed')
else: # Docker
self.check_docker()
if self.config.custom_docker_image:
print_info('Skipping dependency check, using a custom docker image')
else:
command_ppa = ''
if self.config.dependencies_ppa:
command_ppa = 'RUN add-apt-repository {}'.format(' '.join(self.config.dependencies_ppa))
dockerfile = '''
FROM {}
RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate
{}
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean
'''.format(
self.base_docker_image,
command_ppa,
' '.join(dependencies)
).strip()
build = False
if not os.path.exists(self.clickableDir):
os.makedirs(self.clickableDir)
if os.path.exists(self.dockerFile):
with open(self.dockerFile, 'r') as f:
if dockerfile.strip() != f.read().strip():
build = True
else:
build = True
if not build:
command = 'docker images -q {}'.format(self.docker_image)
image_exists = run_subprocess_check_output(command).strip()
build = not image_exists
if build:
with open(self.dockerFile, 'w') as f:
f.write(dockerfile)
self.docker_image = '{}-{}'.format(self.base_docker_image, uuid.uuid4())
with open(self.dockerNameFile, 'w') as f:
f.write(self.docker_image)
print_info('Generating new docker image')
try:
subprocess.check_call(shlex.split('docker build -t {} .'.format(self.docker_image)), cwd=self.clickableDir)
except subprocess.CalledProcessError:
self.clean_clickable()
raise
else:
print_info('Dependencies already setup')
def get_docker_version_string(self):
return run_subprocess_check_output(
"docker version --format '{{.Client.Version}}'")
def run_command(self,
command,
root_user=False,
get_output=False,
use_build_dir=True,
cwd=None,
tty=False,
localhost=False):
wrapped_command = command
cwd = cwd if cwd else os.path.abspath(self.config.root_dir)
if self.config.container_mode:
wrapped_command = 'bash -c "{}"'.format(command)
else: # Docker
self.check_docker()
if ' ' in cwd or ' ' in self.config.build_dir:
raise ClickableException(
'There are spaces in the current path, this will cause errors in the build process'
)
if self.config.first_docker_info:
logger.debug('Using docker container "{}"'.format(
self.docker_image))
self.config.first_docker_info = False
go_config = ''
if self.config.builder == Constants.GO and self.config.gopath:
gopaths = self.config.gopath.split(':')
docker_gopaths = [
'/gopath/path{}'.format(index)
for index in range(len(gopaths))
]
go_config = '-e GOPATH={}'.format(':'.join(docker_gopaths), )
rust_config = ''
if self.config.builder == Constants.RUST and self.config.cargo_home:
logger.info("Caching cargo related files in {}".format(
self.config.cargo_home))
env_vars = self.config.prepare_docker_env_vars()
user = ""
if not root_user:
user = "******".format(os.getuid())
mounts = self.render_mounts(
self.get_docker_mounts(transparent=[cwd]))
wrapped_command = 'docker run {mounts} {env} {go} {rust} {user} -w {cwd} --rm {tty} {network} -i {image} bash -c "{cmd}"'.format(
mounts=mounts,
env=env_vars,
go=go_config,
rust=rust_config,
cwd=self.config.build_dir if use_build_dir else cwd,
user=user,
image=self.docker_image,
cmd=command,
tty="-t" if tty else "",
network='--network="host"' if localhost else "",
)
kwargs = {}
if use_build_dir:
kwargs['cwd'] = self.config.build_dir
if get_output:
return run_subprocess_check_output(shlex.split(wrapped_command),
**kwargs)
else:
subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def run(self, path_arg=None):
'''
Inspired by http://bazaar.launchpad.net/~phablet-team/phablet-tools/trunk/view/head:/phablet-shell
'''
if self.config.ssh:
subprocess.check_call(
shlex.split('ssh phablet@{}'.format(self.config.ssh)))
else:
self.device.check_any_attached()
adb_args = ''
if self.config.device_serial_number:
adb_args = '-s {}'.format(self.config.device_serial_number)
else:
self.device.check_multiple_attached()
output = run_subprocess_check_output(
shlex.split(
'adb {} shell pgrep sshd'.format(adb_args))).split()
if not output:
self.toggle_ssh(on=True)
# Use the usb cable rather than worrying about going over wifi
port = 0
for p in range(2222, 2299):
error_code = run_subprocess_call(shlex.split(
'adb {} forward tcp:{} tcp:22'.format(adb_args, p)),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if error_code == 0:
port = p
break
if port == 0:
raise Exception('Failed to open a port to the device')
# Purge the device host key so that SSH doesn't print a scary warning about it
# (it changes every time the device is reflashed and this is expected)
known_hosts = os.path.expanduser('~/.ssh/known_hosts')
subprocess.check_call(shlex.split('touch {}'.format(known_hosts)))
subprocess.check_call(
shlex.split('ssh-keygen -f {} -R [localhost]:{}'.format(
known_hosts, port)))
id_pub = os.path.expanduser('~/.ssh/id_rsa.pub')
if not os.path.isfile(id_pub):
raise Exception(
'Could not find a ssh public key at "{}", please generate one and try again'
.format(id_pub))
with open(id_pub, 'r') as f:
public_key = f.read().strip()
self.device.run_command('[ -d ~/.ssh ] || mkdir ~/.ssh',
cwd=self.config.cwd)
self.device.run_command('touch ~/.ssh/authorized_keys',
cwd=self.config.cwd)
output = run_subprocess_check_output(
'adb {} shell "grep \\"{}\\" ~/.ssh/authorized_keys"'.format(
adb_args, public_key),
shell=True).strip()
if not output or 'No such file or directory' in output:
print_info('Inserting ssh public key on the connected device')
self.device.run_command(
'echo \"{}\" >>~/.ssh/authorized_keys'.format(public_key),
cwd=self.config.cwd)
self.device.run_command('chmod 700 ~/.ssh',
cwd=self.config.cwd)
self.device.run_command('chmod 600 ~/.ssh/authorized_keys',
cwd=self.config.cwd)
subprocess.check_call(
shlex.split(
'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p {} phablet@localhost'
.format(port)))
self.toggle_ssh(on=False)
def setup_dependencies(self):
if len(self.config.dependencies) > 0:
print_info('Checking dependencies')
if self.config.lxd or self.config.container_mode:
command = 'apt-get install -y --force-yes'
run = False
for dep in self.config.dependencies:
if self.config.arch == 'armhf' and 'armhf' not in dep and not self.config.specificDependencies:
dep = '{}:{}'.format(dep, self.config.arch)
exists = ''
try:
exists = self.run_command(
'dpkg -s {} | grep Status'.format(dep),
get_output=True,
use_dir=False)
except subprocess.CalledProcessError:
exists = ''
if exists.strip() != 'Status: install ok installed':
run = True
command = '{} {}'.format(command, dep)
if run:
self.run_command(command, sudo=True, use_dir=False)
else:
print_info('Dependencies already installed')
else: # Docker
self.check_docker()
# TODO make the path to the dockerfile arch dependent
if self.config.custom_docker_image:
print_info(
'Skipping dependency check, using a custom docker image'
)
else:
dependencies = ''
for dep in self.config.dependencies:
if self.config.arch == 'armhf' and 'armhf' not in dep and not self.config.specificDependencies:
dependencies = '{} {}:{}'.format(
dependencies, dep, self.config.arch)
else:
dependencies = '{} {}'.format(dependencies, dep)
dockerfile = '''
FROM {}
RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate
RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean
'''.format(self.base_docker_image, dependencies).strip()
build = False
if not os.path.exists('.clickable'):
os.makedirs('.clickable')
if os.path.exists('.clickable/Dockerfile'):
with open('.clickable/Dockerfile', 'r') as f:
if dockerfile.strip() != f.read().strip():
build = True
else:
build = True
if not build:
command = 'docker images -q {}'.format(
self.docker_image)
image_exists = run_subprocess_check_output(
command).strip()
build = not image_exists
if build:
with open('.clickable/Dockerfile', 'w') as f:
f.write(dockerfile)
self.docker_image = '{}-{}'.format(
self.base_docker_image, uuid.uuid4())
with open('.clickable/name.txt', 'w') as f:
f.write(self.docker_image)
print_info('Generating new docker image')
try:
subprocess.check_call(shlex.split(
'docker build -t {} .'.format(
self.docker_image)),
cwd='.clickable')
except subprocess.CalledProcessError:
self.clean_clickable()
raise
else:
print_info('Dependencies already setup')
def user_part_of_docker_group(self):
output = run_subprocess_check_output(shlex.split('groups {}'.format(getpass.getuser()))).strip()
# Test for exactly docker in the group list
return (' docker ' in output or output.endswith(' docker') or output.startswith('docker ') or output == 'docker')
