def get_time_zone(self): try: return run_subprocess_check_output( 'timedatectl show -p Timezone --value', stderr=subprocess.DEVNULL) except: logger.debug( 'timedatectl show command failed. Falling back to alternative way to detect timezone...' ) if os.path.exists('/etc/timezone'): with open('/etc/timezone') as host_timezone_file: return host_timezone_file.readline().strip() else: logger.debug( '/etc/timezone does not exist. Falling back to alternative way to detect timezone...' ) try: output = run_subprocess_check_output('timedatectl status') for line in output.splitlines(): line = line.strip() if line.startswith('Time zone:'): start = line.find(':') + 1 end = line.find('(') return line[start:end].strip() except: logger.debug( "timedatctl status method failed to set timezone from host in desktop mode..." ) logger.debug("Falling back to UTC as timezone.") return 'UTC'
def run(self, path_arg=None): ''' Inspired by http://bazaar.launchpad.net/~phablet-team/phablet-tools/trunk/view/head:/phablet-shell ''' if self.config.ssh: subprocess.check_call(shlex.split('ssh phablet@{}'.format(self.config.ssh))) else: self.device.check_any_attached() adb_args = '' if self.config.device_serial_number: adb_args = '-s {}'.format(self.config.device_serial_number) else: self.device.check_multiple_attached() output = run_subprocess_check_output(shlex.split('adb {} shell pgrep sshd'.format(adb_args))).split() if not output: self.toggle_ssh(on=True) # Use the usb cable rather than worrying about going over wifi port = 0 for p in range(2222, 2299): error_code = run_subprocess_call(shlex.split('adb {} forward tcp:{} tcp:22'.format(adb_args, p)), stdout=subprocess.PIPE, stderr=subprocess.PIPE) if error_code == 0: port = p break if port == 0: raise Exception('Failed to open a port to the device') # Purge the device host key so that SSH doesn't print a scary warning about it # (it changes every time the device is reflashed and this is expected) known_hosts = os.path.expanduser('~/.ssh/known_hosts') subprocess.check_call(shlex.split('touch {}'.format(known_hosts))) subprocess.check_call(shlex.split('ssh-keygen -f {} -R [localhost]:{}'.format(known_hosts, port))) id_pub = os.path.expanduser('~/.ssh/id_rsa.pub') if not os.path.isfile(id_pub): raise Exception('Could not find a ssh public key at "{}", please generate one and try again'.format(id_pub)) with open(id_pub, 'r') as f: public_key = f.read().strip() self.device.run_command('[ -d ~/.ssh ] || mkdir ~/.ssh', cwd=self.config.cwd) self.device.run_command('touch ~/.ssh/authorized_keys', cwd=self.config.cwd) output = run_subprocess_check_output('adb {} shell "grep \\"{}\\" ~/.ssh/authorized_keys"'.format(adb_args, public_key), shell=True).strip() if not output or 'No such file or directory' in output: print_info('Inserting ssh public key on the connected device') self.device.run_command('echo \"{}\" >>~/.ssh/authorized_keys'.format(public_key), cwd=self.config.cwd) self.device.run_command('chmod 700 ~/.ssh', cwd=self.config.cwd) self.device.run_command('chmod 600 ~/.ssh/authorized_keys', cwd=self.config.cwd) subprocess.check_call(shlex.split('ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p {} phablet@localhost'.format(port))) self.toggle_ssh(on=False)
def check_docker(self, retries=3): try: run_subprocess_check_output(shlex.split('docker ps'), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: retries -= 1 if retries <= 0: raise e self.start_docker() time.sleep(3) # Give it a sec to boot up self.check_docker(retries)
def check_docker(self, retries=3): if self.needs_setup(): self.setup_docker() try: run_subprocess_check_output(shlex.split('docker ps'), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: retries -= 1 if retries <= 0: raise e self.start_docker() time.sleep(3) # Give it a sec to boot up self.check_docker(retries)
def pull_files(self, files, dst_parent): os.makedirs(dst_parent, exist_ok=True) if self.config.container_mode: for f in files: dst_path = os.path.join(dst_parent, os.path.basename(f)) if os.path.isdir(f): if os.path.exists(dst_path): shutil.rmtree(dst_path) shutil.copytree(f, dst_path) else: if os.path.exists(dst_path): os.remove(dst_path) shutil.copy(f, dst_parent, follow_symlinks=False) else: # Docker mounts = self.render_mounts( self.get_docker_mounts(transparent=[self.config.root_dir])) command_create = 'docker create {} {}'.format( mounts, self.docker_image) container = run_subprocess_check_output(command_create).strip() for f in files: command_copy = 'docker cp {}:{} {}'.format( container, f, dst_parent) run_subprocess_check_call(command_copy) command_remove = 'docker rm {}'.format(container) run_subprocess_check_call(command_remove, stdout=subprocess.DEVNULL)
def setup_docker(self): self.check_command('docker') self.start_docker() group_exists = False with open('/etc/group', 'r') as f: lines = f.readlines() for line in lines: if line.startswith('docker:'): group_exists = True if not group_exists: print_info('Asking for root to create docker group') subprocess.check_call(shlex.split('sudo groupadd docker')) output = run_subprocess_check_output( shlex.split('groups {}'.format(getpass.getuser()))).strip() # Test for exactly docker in the group list if ' docker ' in output or output.endswith( ' docker') or output.startswith( 'docker ') or output == 'docker': print_info('Setup has already been completed') else: print_info( 'Asking for root to add the current user to the docker group') subprocess.check_call( shlex.split('sudo usermod -aG docker {}'.format( getpass.getuser()))) print_info('Log out or restart to apply changes')
def check_lxd(self): name = 'clickable-{}'.format(self.config.build_arch) status = '' try: status = run_subprocess_check_output(shlex.split('usdk-target status {}'.format(name)), stderr=subprocess.STDOUT) status = json.loads(status)['status'] except subprocess.CalledProcessError as e: if e.output.strip() == 'error: Could not connect to the LXD server.' or 'Can\'t establish a working socket connection' in e.output.strip(): started = self.start_lxd() if started: status = 'Running' # Pretend it's started, but we will call this function again to check if it's actually ok time.sleep(3) # Give it a sec to boot up self.check_lxd() else: raise Exception('LXD is not running, please start it') elif e.output.strip() == 'error: Could not query container status. error: not found': raise Exception('No lxd container exists to build in, please run `clickable setup-lxd`') else: print(e.output) raise e if status != 'Running': print_info('Going to start lxd container "{}"'.format(name)) subprocess.check_call(shlex.split('lxc start {}'.format(name)))
def user_part_of_docker_group(self): output = run_subprocess_check_output( shlex.split('groups {}'.format(getpass.getuser()))).strip() # Test for exactly docker in the group list return (' docker ' in output or output.endswith(' docker') or output.startswith('docker ') or output == 'docker')
def check_lxd(self): name = 'clickable-{}'.format(self.config.build_arch) status = '' try: status = run_subprocess_check_output(shlex.split( 'usdk-target status {}'.format(name)), stderr=subprocess.STDOUT) status = json.loads(status)['status'] except subprocess.CalledProcessError as e: if e.output.strip( ) == 'error: Could not connect to the LXD server.' or 'Can\'t establish a working socket connection' in e.output.strip( ): started = self.start_lxd() if started: status = 'Running' # Pretend it's started, but we will call this function again to check if it's actually ok time.sleep(3) # Give it a sec to boot up self.check_lxd() else: raise Exception('LXD is not running, please start it') elif e.output.strip( ) == 'error: Could not query container status. error: not found': raise Exception( 'No lxd container exists to build in, please run `clickable setup-lxd`' ) else: print(e.output) raise e if status != 'Running': print_info('Going to start lxd container "{}"'.format(name)) subprocess.check_call(shlex.split('lxc start {}'.format(name)))
def run_container_command(self, command, force_lxd=False, sudo=False, get_output=False, use_dir=True, cwd=None): wrapped_command = command cwd = cwd if cwd else self.cwd if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) elif force_lxd or self.config.lxd: self.check_lxd() target_command = 'exec' if sudo: target_command = 'maint' if use_dir: command = 'cd {}; {}'.format(self.config.dir, command) wrapped_command = 'usdk-target {} clickable-{} -- bash -c "{}"'.format( target_command, self.build_arch, command) elif self.config.chroot: chroot_command = 'run' if sudo: chroot_command = 'maint' wrapped_command = 'click chroot -a {} -f {} {} bash -c "{}"'.format( self.build_arch, self.config.sdk, chroot_command, command) else: # Docker self.check_docker() go_config = '' if self.config.gopath: go_config = '-v {}:/gopath -e GOPATH=/gopath'.format( self.config.gopath) wrapped_command = 'docker run -v {}:{} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format( cwd, cwd, go_config, self.config.dir if use_dir else cwd, os.getuid(), self.docker_image, command, ) kwargs = {} if use_dir: kwargs['cwd'] = self.config.dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def is_met(self): if not is_program_installed('nvidia-smi'): return False modules = run_subprocess_check_output('lsmod').splitlines() for m in modules: if m.split(' ', 1)[0] == 'nvidia': return True return False
def restore_cached_container(self): with open(self.docker_name_file, 'r') as f: cached_container = f.read().strip() if not image_exists(cached_container): logger.warning("Cached container does not exist anymore") return command_base = 'docker images -q {}'.format(self.base_docker_image) command_cached = 'docker history -q {}'.format(cached_container) hash_base = run_subprocess_check_output(command_base).strip() history_cached = run_subprocess_check_output( command_cached).strip() if hash_base in history_cached: logger.debug("Found cached container") self.docker_image = cached_container else: logger.warning("Found outdated container")
def restore_cached_image(self): if not os.path.exists(self.docker_name_file): return with open(self.docker_name_file, 'r') as f: cached_image = None cached_base_image = None try: image_file = json.load(f) cached_image = image_file.get('name', None) cached_base_image = image_file.get('base_image', None) except ValueError: pass if not cached_image: logger.warning("Cached image file is invalid") return if not image_exists(cached_image): logger.warning("Cached container does not exist anymore") return if self.base_docker_image != cached_base_image: logger.warning("Cached image has a different base image") self.check_docker() command_base = 'docker images -q {}'.format(self.base_docker_image) command_cached = 'docker history -q {}'.format(cached_image) hash_base = run_subprocess_check_output(command_base).strip() history_cached = run_subprocess_check_output( command_cached).strip() if hash_base in history_cached: logger.debug("Found cached container") self.docker_image = cached_image else: logger.warning("Cached container is outdated")
def lxd_container_exists(self): name = 'clickable-{}'.format(self.config.build_arch) # Check for existing container existing = run_subprocess_check_output(shlex.split('{} list'.format(self.usdk_target))) existing = json.loads(existing) found = False for container in existing: if container['name'] == name: found = True return found
def check_docker(self, retries=3): if not self.docker_mode: raise ClickableException( "Container was not initialized with Container Mode. This seems to be a bug in Clickable." ) if self.needs_docker_setup(): self.setup_docker() try: run_subprocess_check_output(shlex.split('docker ps'), stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: retries -= 1 if retries <= 0: raise ClickableException( "Couldn't check docker. If you just installed Clickable you may need to reboot once." ) self.start_docker() time.sleep(3) # Give it a sec to boot up self.check_docker(retries)
def run(self, path_arg=None): self.container.check_docker() command = 'docker pull {}'.format(self.container.base_docker_image) run_subprocess_check_call(command) if 'armhf' in self.container.base_docker_image: image = self.container.base_docker_image.replace('armhf', 'amd64') command = 'docker images -q {}'.format(image) image_exists = run_subprocess_check_output(command).strip() if image_exists: command = 'docker pull {}'.format(image) run_subprocess_check_call(command)
def lxd_container_exists(self): name = 'clickable-{}'.format(self.config.build_arch) # Check for existing container existing = run_subprocess_check_output( shlex.split('{} list'.format(self.usdk_target))) existing = json.loads(existing) found = False for container in existing: if container['name'] == name: found = True return found
def is_dockerfile_outdated(self, dockerfile_content): if self.docker_image == self.base_docker_image: return True if not os.path.exists(self.clickable_dir): return True if not os.path.exists(self.docker_file): return True with open(self.docker_file, 'r') as f: if dockerfile_content.strip() != f.read().strip(): return True command = 'docker images -q {}'.format(self.docker_image) image_exists = run_subprocess_check_output(command).strip() return not image_exists
def detect_devices(self): output = run_subprocess_check_output( shlex.split('adb devices -l')).strip() devices = [] for line in output.split('\n'): if 'device' in line and 'devices' not in line: device = line.split(' ')[0] for part in line.split(' '): if part.startswith('model:'): device = '{} - {}'.format( device, part.replace('model:', '').replace('_', ' ').strip()) devices.append(device) return devices
def check_base_image_version(self): if not self.minimum_version: return if not image_exists(self.docker_image): return version = 0 try: format_string = '{{ index .Config.Labels "image_version"}}' command = "docker inspect --format '{}' {}".format( format_string, self.docker_image) logger.debug( 'Checking docker image version via: {}'.format(command)) version_string = run_subprocess_check_output(command) version = int(version_string) except (ValueError, subprocess.CalledProcessError): logger.warn("Could not read the image version from the container") if version < self.minimum_version: raise ClickableException( 'This version of Clickable requires Clickable docker image {} in version {} or higher (found version {}). Please run "clickable update" to update your local images.' .format(self.docker_image, self.minimum_version, version))
def run_command(self, command, force_lxd=False, sudo=False, get_output=False, use_dir=True, cwd=None): wrapped_command = command cwd = cwd if cwd else self.config.cwd if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) elif force_lxd or self.config.lxd: self.check_lxd() target_command = 'exec' if sudo: target_command = 'maint' if use_dir: command = 'cd {}; {}'.format(self.config.dir, command) wrapped_command = 'usdk-target {} clickable-{} -- bash -c "{}"'.format( target_command, self.config.build_arch, command) else: # Docker self.check_docker() if ' ' in cwd or ' ' in self.config.dir: raise Exception( 'There are spaces in the current path, this will cause errors in the build process' ) if self.config.first_docker_info: print_info('Using docker container "{}"'.format( self.docker_image)) self.config.first_docker_info = False go_config = '' if self.config.gopath: go_config = '-v {}:/gopath -e GOPATH=/gopath'.format( self.config.gopath) wrapped_command = 'docker run -v {}:{} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format( cwd, cwd, go_config, self.config.dir if use_dir else cwd, os.getuid(), self.docker_image, command, ) kwargs = {} if use_dir: kwargs['cwd'] = self.config.dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def run_command(self, command, force_lxd=False, sudo=False, get_output=False, use_dir=True, cwd=None): wrapped_command = command cwd = cwd if cwd else os.path.abspath(self.config.root_dir) if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) elif force_lxd or self.config.lxd: self.check_lxd() target_command = 'exec' if sudo: target_command = 'maint' if use_dir: command = 'cd {}; {}'.format(self.config.build_dir, command) wrapped_command = 'usdk-target {} clickable-{} -- bash -c "{}"'.format(target_command, self.config.build_arch, command) else: # Docker self.check_docker() if ' ' in cwd or ' ' in self.config.build_dir: raise Exception('There are spaces in the current path, this will cause errors in the build process') if self.config.first_docker_info: print_info('Using docker container "{}"'.format(self.docker_image)) self.config.first_docker_info = False go_config = '' if self.config.gopath: gopaths = self.config.gopath.split(':') docker_gopaths = [] go_configs = [] for (index, path) in enumerate(gopaths): go_configs.append('-v {}:/gopath/path{}:Z'.format(path, index)) docker_gopaths.append('/gopath/path{}'.format(index)) go_config = '{} -e GOPATH={}'.format( ' '.join(go_configs), ':'.join(docker_gopaths), ) rust_config = '' if self.config.config['template'] == Config.RUST and self.config.cargo_home: cargo_registry = os.path.join(self.config.cargo_home, 'registry') cargo_git = os.path.join(self.config.cargo_home, 'git') os.makedirs(cargo_registry, exist_ok=True) os.makedirs(cargo_git, exist_ok=True) rust_config = '-v {}:/opt/rust/cargo/registry:Z -v {}:/opt/rust/cargo/git:Z'.format( cargo_registry, cargo_git, ) wrapped_command = 'docker run -v {}:{}:Z {} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format( cwd, cwd, go_config, rust_config, self.config.build_dir if use_dir else cwd, os.getuid(), self.docker_image, command, ) kwargs = {} if use_dir: kwargs['cwd'] = self.config.build_dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def setup_dependencies(self, force_build=False): if self.config.dependencies_build or self.config.dependencies_target: logger.debug('Checking dependencies') dependencies = self.config.dependencies_build for dep in self.config.dependencies_target: if ':' in dep: dependencies.append(dep) else: dependencies.append('{}:{}'.format(dep, self.config.arch)) if self.config.container_mode: self.run_command('apt-get update', sudo=True, use_dir=False) command = 'apt-get install -y --force-yes' run = False for dep in dependencies: exists = '' try: exists = self.run_command( 'dpkg -s {} | grep Status'.format(dep), get_output=True, use_dir=False) except subprocess.CalledProcessError: exists = '' if exists.strip() != 'Status: install ok installed': run = True command = '{} {}'.format(command, dep) if run: self.run_command(command, sudo=True, use_dir=False) else: logger.debug('Dependencies already installed') else: self.check_docker() if self.config.custom_docker_image: logger.warning( 'Skipping dependency check, using a custom docker image' ) else: command_ppa = '' if self.config.dependencies_ppa: command_ppa = 'RUN add-apt-repository {}'.format( ' '.join(self.config.dependencies_ppa)) dockerfile = ''' FROM {} RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate {} RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean '''.format(self.base_docker_image, command_ppa, ' '.join(dependencies)).strip() build = force_build if not os.path.exists(self.clickable_dir): os.makedirs(self.clickable_dir) if self.docker_image != self.base_docker_image and os.path.exists( self.docker_file): with open(self.docker_file, 'r') as f: if dockerfile.strip() != f.read().strip(): build = True else: build = True if not build: command = 'docker images -q {}'.format( self.docker_image) image_exists = run_subprocess_check_output( command).strip() build = not image_exists if build: with open(self.docker_file, 'w') as f: f.write(dockerfile) self.docker_image = '{}-{}'.format( self.base_docker_image, uuid.uuid4()) with open(self.docker_name_file, 'w') as f: f.write(self.docker_image) logger.debug('Generating new docker image') try: subprocess.check_call(shlex.split( 'docker build -t {} .'.format( self.docker_image)), cwd=self.clickable_dir) except subprocess.CalledProcessError: self.clean_clickable() raise else: logger.debug('Dependencies already setup')
def run_command(self, command, sudo=False, get_output=False, use_dir=True, cwd=None): wrapped_command = command cwd = cwd if cwd else os.path.abspath(self.config.root_dir) if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) else: # Docker self.check_docker() if ' ' in cwd or ' ' in self.config.build_dir: raise ClickableException( 'There are spaces in the current path, this will cause errors in the build process' ) if self.config.first_docker_info: logger.debug('Using docker container "{}"'.format( self.docker_image)) self.config.first_docker_info = False go_config = '' if self.config.gopath: gopaths = self.config.gopath.split(':') docker_gopaths = [] go_configs = [] for (index, path) in enumerate(gopaths): go_configs.append('-v {}:/gopath/path{}:Z'.format( path, index)) docker_gopaths.append('/gopath/path{}'.format(index)) go_config = '{} -e GOPATH={}'.format( ' '.join(go_configs), ':'.join(docker_gopaths), ) rust_config = '' if self.config.config[ 'template'] == Config.RUST and self.config.cargo_home: logger.info("Caching cargo related files in {}".format( self.config.cargo_home)) cargo_registry = os.path.join(self.config.cargo_home, 'registry') cargo_git = os.path.join(self.config.cargo_home, 'git') cargo_package_cache_lock = os.path.join( self.config.cargo_home, '.package-cache') os.makedirs(cargo_registry, exist_ok=True) os.makedirs(cargo_git, exist_ok=True) # create .package-cache if it doesn't exist with open(cargo_package_cache_lock, "a"): pass rust_config = '-v {}:/opt/rust/cargo/registry:Z -v {}:/opt/rust/cargo/git:Z -v {}:/opt/rust/cargo/.package-cache'.format( cargo_registry, cargo_git, cargo_package_cache_lock, ) env_vars = self.config.prepare_docker_env_vars() wrapped_command = 'docker run -v {}:{}:Z {} {} {} -w {} -u {} -e HOME=/tmp --rm -i {} bash -c "{}"'.format( cwd, cwd, env_vars, go_config, rust_config, self.config.build_dir if use_dir else cwd, os.getuid(), self.docker_image, command, ) kwargs = {} if use_dir: kwargs['cwd'] = self.config.build_dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def setup_dependencies(self): if self.config.dependencies_build or self.config.dependencies_target: print_info('Checking dependencies') dependencies = self.config.dependencies_build for dep in self.config.dependencies_target: if ':' in dep: dependencies.append(dep) else: dependencies.append('{}:{}'.format(dep, self.config.arch)) if self.config.lxd or self.config.container_mode: self.run_command('apt-get update', sudo=True, use_dir=False) command = 'apt-get install -y --force-yes' run = False for dep in dependencies: exists = '' try: exists = self.run_command('dpkg -s {} | grep Status'.format(dep), get_output=True, use_dir=False) except subprocess.CalledProcessError: exists = '' if exists.strip() != 'Status: install ok installed': run = True command = '{} {}'.format(command, dep) if run: self.run_command(command, sudo=True, use_dir=False) else: print_info('Dependencies already installed') else: # Docker self.check_docker() if self.config.custom_docker_image: print_info('Skipping dependency check, using a custom docker image') else: command_ppa = '' if self.config.dependencies_ppa: command_ppa = 'RUN add-apt-repository {}'.format(' '.join(self.config.dependencies_ppa)) dockerfile = ''' FROM {} RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate {} RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean '''.format( self.base_docker_image, command_ppa, ' '.join(dependencies) ).strip() build = False if not os.path.exists(self.clickableDir): os.makedirs(self.clickableDir) if os.path.exists(self.dockerFile): with open(self.dockerFile, 'r') as f: if dockerfile.strip() != f.read().strip(): build = True else: build = True if not build: command = 'docker images -q {}'.format(self.docker_image) image_exists = run_subprocess_check_output(command).strip() build = not image_exists if build: with open(self.dockerFile, 'w') as f: f.write(dockerfile) self.docker_image = '{}-{}'.format(self.base_docker_image, uuid.uuid4()) with open(self.dockerNameFile, 'w') as f: f.write(self.docker_image) print_info('Generating new docker image') try: subprocess.check_call(shlex.split('docker build -t {} .'.format(self.docker_image)), cwd=self.clickableDir) except subprocess.CalledProcessError: self.clean_clickable() raise else: print_info('Dependencies already setup')
def get_docker_version_string(self): return run_subprocess_check_output( "docker version --format '{{.Client.Version}}'")
def run_command(self, command, root_user=False, get_output=False, use_build_dir=True, cwd=None, tty=False, localhost=False): wrapped_command = command cwd = cwd if cwd else os.path.abspath(self.config.root_dir) if self.config.container_mode: wrapped_command = 'bash -c "{}"'.format(command) else: # Docker self.check_docker() if ' ' in cwd or ' ' in self.config.build_dir: raise ClickableException( 'There are spaces in the current path, this will cause errors in the build process' ) if self.config.first_docker_info: logger.debug('Using docker container "{}"'.format( self.docker_image)) self.config.first_docker_info = False go_config = '' if self.config.builder == Constants.GO and self.config.gopath: gopaths = self.config.gopath.split(':') docker_gopaths = [ '/gopath/path{}'.format(index) for index in range(len(gopaths)) ] go_config = '-e GOPATH={}'.format(':'.join(docker_gopaths), ) rust_config = '' if self.config.builder == Constants.RUST and self.config.cargo_home: logger.info("Caching cargo related files in {}".format( self.config.cargo_home)) env_vars = self.config.prepare_docker_env_vars() user = "" if not root_user: user = "******".format(os.getuid()) mounts = self.render_mounts( self.get_docker_mounts(transparent=[cwd])) wrapped_command = 'docker run {mounts} {env} {go} {rust} {user} -w {cwd} --rm {tty} {network} -i {image} bash -c "{cmd}"'.format( mounts=mounts, env=env_vars, go=go_config, rust=rust_config, cwd=self.config.build_dir if use_build_dir else cwd, user=user, image=self.docker_image, cmd=command, tty="-t" if tty else "", network='--network="host"' if localhost else "", ) kwargs = {} if use_build_dir: kwargs['cwd'] = self.config.build_dir if get_output: return run_subprocess_check_output(shlex.split(wrapped_command), **kwargs) else: subprocess.check_call(shlex.split(wrapped_command), **kwargs)
def run(self, path_arg=None): ''' Inspired by http://bazaar.launchpad.net/~phablet-team/phablet-tools/trunk/view/head:/phablet-shell ''' if self.config.ssh: subprocess.check_call( shlex.split('ssh phablet@{}'.format(self.config.ssh))) else: self.device.check_any_attached() adb_args = '' if self.config.device_serial_number: adb_args = '-s {}'.format(self.config.device_serial_number) else: self.device.check_multiple_attached() output = run_subprocess_check_output( shlex.split( 'adb {} shell pgrep sshd'.format(adb_args))).split() if not output: self.toggle_ssh(on=True) # Use the usb cable rather than worrying about going over wifi port = 0 for p in range(2222, 2299): error_code = run_subprocess_call(shlex.split( 'adb {} forward tcp:{} tcp:22'.format(adb_args, p)), stdout=subprocess.PIPE, stderr=subprocess.PIPE) if error_code == 0: port = p break if port == 0: raise Exception('Failed to open a port to the device') # Purge the device host key so that SSH doesn't print a scary warning about it # (it changes every time the device is reflashed and this is expected) known_hosts = os.path.expanduser('~/.ssh/known_hosts') subprocess.check_call(shlex.split('touch {}'.format(known_hosts))) subprocess.check_call( shlex.split('ssh-keygen -f {} -R [localhost]:{}'.format( known_hosts, port))) id_pub = os.path.expanduser('~/.ssh/id_rsa.pub') if not os.path.isfile(id_pub): raise Exception( 'Could not find a ssh public key at "{}", please generate one and try again' .format(id_pub)) with open(id_pub, 'r') as f: public_key = f.read().strip() self.device.run_command('[ -d ~/.ssh ] || mkdir ~/.ssh', cwd=self.config.cwd) self.device.run_command('touch ~/.ssh/authorized_keys', cwd=self.config.cwd) output = run_subprocess_check_output( 'adb {} shell "grep \\"{}\\" ~/.ssh/authorized_keys"'.format( adb_args, public_key), shell=True).strip() if not output or 'No such file or directory' in output: print_info('Inserting ssh public key on the connected device') self.device.run_command( 'echo \"{}\" >>~/.ssh/authorized_keys'.format(public_key), cwd=self.config.cwd) self.device.run_command('chmod 700 ~/.ssh', cwd=self.config.cwd) self.device.run_command('chmod 600 ~/.ssh/authorized_keys', cwd=self.config.cwd) subprocess.check_call( shlex.split( 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p {} phablet@localhost' .format(port))) self.toggle_ssh(on=False)
def setup_dependencies(self): if len(self.config.dependencies) > 0: print_info('Checking dependencies') if self.config.lxd or self.config.container_mode: command = 'apt-get install -y --force-yes' run = False for dep in self.config.dependencies: if self.config.arch == 'armhf' and 'armhf' not in dep and not self.config.specificDependencies: dep = '{}:{}'.format(dep, self.config.arch) exists = '' try: exists = self.run_command( 'dpkg -s {} | grep Status'.format(dep), get_output=True, use_dir=False) except subprocess.CalledProcessError: exists = '' if exists.strip() != 'Status: install ok installed': run = True command = '{} {}'.format(command, dep) if run: self.run_command(command, sudo=True, use_dir=False) else: print_info('Dependencies already installed') else: # Docker self.check_docker() # TODO make the path to the dockerfile arch dependent if self.config.custom_docker_image: print_info( 'Skipping dependency check, using a custom docker image' ) else: dependencies = '' for dep in self.config.dependencies: if self.config.arch == 'armhf' and 'armhf' not in dep and not self.config.specificDependencies: dependencies = '{} {}:{}'.format( dependencies, dep, self.config.arch) else: dependencies = '{} {}'.format(dependencies, dep) dockerfile = ''' FROM {} RUN echo set debconf/frontend Noninteractive | debconf-communicate && echo set debconf/priority critical | debconf-communicate RUN apt-get update && apt-get install -y --force-yes --no-install-recommends {} && apt-get clean '''.format(self.base_docker_image, dependencies).strip() build = False if not os.path.exists('.clickable'): os.makedirs('.clickable') if os.path.exists('.clickable/Dockerfile'): with open('.clickable/Dockerfile', 'r') as f: if dockerfile.strip() != f.read().strip(): build = True else: build = True if not build: command = 'docker images -q {}'.format( self.docker_image) image_exists = run_subprocess_check_output( command).strip() build = not image_exists if build: with open('.clickable/Dockerfile', 'w') as f: f.write(dockerfile) self.docker_image = '{}-{}'.format( self.base_docker_image, uuid.uuid4()) with open('.clickable/name.txt', 'w') as f: f.write(self.docker_image) print_info('Generating new docker image') try: subprocess.check_call(shlex.split( 'docker build -t {} .'.format( self.docker_image)), cwd='.clickable') except subprocess.CalledProcessError: self.clean_clickable() raise else: print_info('Dependencies already setup')
def user_part_of_docker_group(self): output = run_subprocess_check_output(shlex.split('groups {}'.format(getpass.getuser()))).strip() # Test for exactly docker in the group list return (' docker ' in output or output.endswith(' docker') or output.startswith('docker ') or output == 'docker')