def run_interactive_build_command(self, command): verbose(f"Docker host - run interactive command '{command}'") if dryrun(): print(f"dry: (interactive-in-container) {command}") else: try: socket = self.container.exec_run( cmd=command, tty=True, stdin=True, socket=True, demux=False, workdir=self.config.project_dir_container) socket.output._sock.send(b'export PSADD="(c*r)"\n') while True: r, w, e = select.select([sys.stdin, socket.output._sock], [], [sys.stdin, socket.output._sock]) if sys.stdin in r: d = os.read(sys.stdin.fileno(), 10240) socket.output._sock.send(d) elif socket.output._sock in r: data = socket.output._sock.recv(16384) os.write(sys.stdout.fileno(), data) if sys.stdin in e or socket.output._sock in e: break # leave the loop except Exception as err: verbose(f"Exception: {err}")
def __init__(self, targets_file=None): verbose(f"Target Builder") super().__init__() self.targets = get_targets(targets_file) self.do_packageindex = get_value_with_default( ["build", "packageindex"]) self.context = BatchContext(self.source_line)
def create_dir_and_dockerfile( yocto_bitbaker_image="almedso/yocto-bitbaker:latest", yocto_user_home="/home/yocto"): """ create a temporary directory and add a Dockerfile to create a privatized container """ uid = os.getuid() gid = os.getgid() verbose(f"Inject uid {uid} and gid {gid}") dockerfile = f""" FROM {yocto_bitbaker_image} RUN pip3 install --upgrade ronto RUN groupadd --gid {gid} yocto || true && \ useradd --uid {uid} --gid {gid} --home {yocto_user_home} \ --create-home --shell /bin/bash yocto USER yocto """ dir = tempfile.mkdtemp() filename = os.path.join(dir, "Dockerfile") with open(filename, "w") as f: f.write(dockerfile) return dir
def env_val(key, value): env = os.getenv(key) if env: verbose(f"Read from environment {key}: {env}") return env else: return value
def start_container(self): if dryrun(): print(f"dry: Start container: {self.container_name}") else: verbose(f"Docker container status: {self.container.status}") if self.container.status != "running": verbose(f"Start docker container") self.container.start()
def run_init(): """ Source the init script once to place/ update build dir structure i.e. create build dir, conf dir and add local.conf, bblayer.conf """ source_line = init_to_source_in() verbose(f"Run init: {source_line[:-1]}") run_cmd(["bash", "-c", source_line])
def init_process(args): verbose("Process init command") clean_init(rebuild_conf=args.rebuild_conf, clean_conf_dir=args.clean_conf, clean_build_dir=args.clean_build) run_init() siteconf = SiteConfigHandler() siteconf.handle()
def __init__(self): script = get_init_script() self.build_dir = get_init_build_dir() # There is no need to source something with templatedir # this is done potentially in init command source_line = "source " + script + " " + self.build_dir verbose(f"Builder init sourcing: {source_line}") self.source_line = source_line
def publish_target(target): publish_dir = publish_image_dir() if publish_dir != None: # make sure directories are available os.makedirs(publish_dir, exist_ok=True) if 'publish' in target: for image_machine in image_files(target): verbose(f"Publish target {os.path.basename(image_machine)}") run_cmd(['cp', '-fL', image_machine, publish_dir])
def check_version(version): if version != None: try: verbose(f"Check rontofile version {version}") int_version = int(version) if int_version > 1: raise VersionError() except ValueError: raise VersionError()
def build_process(args): verbose("Process build command") if args.interactive and not args.list_targets: builder = InteractiveBuilder() else: builder = TargetBuilder() if args.list_targets: builder.list_targets() else: builder.build()
def fetch(cls, force=None): if cls.url != "": verbose(f"Init repo from {cls.url}") run_cmd([ "repo", "init", "-u", cls.url, "-m", cls.manifest, "-b", cls.branch ]) force_sync = "--force-sync" if force else "" verbose(f"Sync repo {force_sync}") run_cmd(["repo", "sync", force_sync])
def build(self): for target in self.targets: # print instead of verbose since bitbake is verbose anyway print(f"****************************************************") print(f"* Build {target['image']} for {target['machine']}") print(f"****************************************************") self.context.run_context( f"MACHINE={target['machine']} bitbake {target['image']}") if self.do_packageindex: verbose("Do package index") self.context.run_context("bitbake package-index")
def verify_target_specification(raw_targets): verbose("Verify target specifications") targets = [] if (raw_targets and isinstance(raw_targets, list)): for target in raw_targets: if (isinstance(target, dict) and "machine" in target and isinstance(target["machine"], str) and "image" in target and isinstance(target["image"], str)): targets.append(target) return targets
def inject_command(self, command): if command != '': verbose(f"Run command: {command}") composed_cmd = self.source_line + '\n' self.process.stdin.write(composed_cmd.encode()) composed_cmd = command + '\n' self.process.stdin.write(composed_cmd.encode()) composed_cmd = 'exit\n' self.process.stdin.write(composed_cmd.encode()) composed_cmd = 'exit\n' self.process.stdin.write(composed_cmd.encode())
def build_privatized_docker_image(self): if dryrun(): print(f"dry: Build or get privatized docker image: " \ f"{self.config.privatized_image}") else: try: image_label = self.config.privatized_image + ":latest" image = self.docker.images.get(image_label) verbose(f"Privatized image {image_label} exists" \ " - no need to build") except docker.errors.ImageNotFound as _err: self._build_privatized_docker_image()
def update_defaults(): """ Section defaults to deal with, like: defaults: FOO: 'foo' BAR: 'bar' """ global variables_ verbose("Update default variables") if "defaults" in model_ and isinstance(model_["defaults"], dict): for default in model_["defaults"]: variables_[default] = env_val(default, model_["defaults"][default])
def _build_privatized_docker_image(self): verbose("Build privatized docker image") privatized_docker_image = self.config.get_privatized_image() yocto_docker_image = self.config.get_image() dir = create_dir_and_dockerfile(yocto_docker_image, self.yocto_user_home) if dryrun(): with open(os.path.join(dir, "Dockerfile"), "r") as f: print("dry: privatizing Dockerfile" + f.read()) run_cmd(["docker", "build", "-t", privatized_docker_image, dir]) os.remove(os.path.join(dir, "Dockerfile")) # cleanup Dockerfile os.rmdir(dir) # cleanup temporary directory
def get_targets(targets_file): """ Get list of targets plus inspection of them """ raw_targets = get_targets_from_yaml_file(targets_file) if not raw_targets: verbose("Check for targets directly defined in 'ronto.yml'") raw_targets = get_value(["build", "targets"]) targets = verify_target_specification(raw_targets) if len(targets) == 0: verbose(" No verified target found -> use default target") # Add a default machine/image combination as of yocto docu # getting started section. targets.append({"machine": "qemux86", "image": "core-image-sato"}) return targets
def run_context(self, command): if dryrun(): print(f"dry - Run build command: {command}") else: try: self.process = subprocess.Popen( "bash", stdin=subprocess.PIPE, stderr=subprocess.STDOUT, ) verbose(f"Start Bash session: Pid {self.process.pid}") self.inject_command(command) self.process.communicate() except subprocess.SubprocessError: pass verbose(f"Stop bash session: Pid {self.process.pid}")
def run_context(self): # carridge return is required due to change terminal discipline verbose(f"run context\r") while self.process.poll() is None: r, w, e = select.select([sys.stdin, self.master_fd], [], [sys.stdin, self.master_fd]) if sys.stdin in r: d = os.read(sys.stdin.fileno(), 10240) os.write(self.master_fd, d) elif self.master_fd in r: o = os.read(self.master_fd, 10240) if o: os.write(sys.stdout.fileno(), o) if sys.stdin in e or self.master_fd in e: break termios.tcsetattr(sys.stdin, termios.TCSADRAIN, self.old_tty) verbose(f"Stop bash session: Pid {self.process.pid}")
def process(args): docker = docker_factory() # only on docker host successful/usefull if docker: docker.build_privatized_docker_image() docker.create_container() docker.start_container() docker.run_command(args.cmd, args.interactive) docker.stop_container() if args.rm_container: docker.remove_container() if args.rm_priv_image: docker.remove_privatized_image() if args.rm_all: docker.remove_all() else: verbose("No docker environment")
def handle(self): """ Update site.conf if overwrite and file available Create site.conf if source file available """ dest_config_dir = os.path.join(os.getcwd(), self.build_dir, "conf") dest_site_conf_file = os.path.join(dest_config_dir, "site.conf") if not os.path.isfile(dest_site_conf_file): # site.conf file does not exit (Create scenario) src_site_conf_file = os.path.join(os.getcwd(), self.sitefile) if os.path.isfile(src_site_conf_file): verbose(f"Create site.conf from: {src_site_conf_file}") if dryrun(): print( f"copy {src_site_conf_file} to {dest_site_conf_file}") else: os.makedirs(dest_config_dir, exist_ok=True) copyfile(src_site_conf_file, dest_site_conf_file)
def run_command(self, command, interactive_flag=False): # cleanup the ronto command path if isinstance(command, list): if 'ronto' in command[0]: command[0] = 'ronto' # get rid of of host local path if isinstance(command, str): p = re.compile('^([\w/]*ronto)') command = p.sub('ronto', command) if interactive_flag \ or '-i' in command \ or '--interactive' in command: # in works on lists and on strings self.run_interactive_build_command(command) else: self.run_batch_command(command) if isinstance(command, list): command = ' '.join(command) verbose(f"Docker command '{command}' finished - returned to host")
def get_targets_from_yaml_file(targets_file): """ Read targets from file (either input or defined in ronto.yml ) @target_file: relative path of targets file from project directory or None @returns None of structure of the file """ if not targets_file: targets_file = get_value_with_default(['build', 'targets_file']) if targets_file: verbose(f"Read targets from file: {targets_file}") try: with open(targets_file) as file: return yaml.load(file, Loader=yaml.FullLoader) except FileNotFoundError: print(f"File with target specifications not found -> fall back", file=sys.stderr) return None return None
def __init__(cls): if exists(["docker"]): verbose("Docker configuration found") cls._use_docker = True cls.image = get_value_with_default(["docker", "image"], "almedso/yocto-bitbaker:latest") cls.privatized_image = get_value_with_default( ["docker", "privatized_image"], "my-yocto-bitbaker") cls.project_dir_container = get_value_with_default( ["docker", "project_dir"], "/yocto/root") cls.cache_dir_host = get_value_with_default( ["docker", "cache_dir", "host"], os.path.abspath(os.path.join(PROJECT_DIR_HOST, '..', 'cache'))) cls.cache_dir_container = get_value_with_default( ["docker", "cache_dir", "container"], "/yocto/cache") cls.publish_dir_host = get_value_with_default( ["docker", "publish_dir", "host"], "") cls.publish_dir_container = get_value_with_default( ["docker", "publish_dir", "container"], "/yocto/publish")
def __init__(cls): """ Initialize from Rontofile: Rontofile syntax is: repo: url: git://host/git-manifest-repo.git manifest: release-xyz.xml branch: master """ if 'repo' in get_model(): # skip totally if repo is not set. is_command_available_or_exit(["repo", "--version"]) verbose(f"Config base: Google manifest repository") cls.url = get_value(['repo', 'url']) if cls.url == None: print("repo URL cannot be determined", file=sys.stderr) sys.exit(1) cls.branch = get_value_with_default(["repo", "branch"], 'master') cls.manifest = get_value_with_default(["repo", "manifest"], 'default.xml')
def run_batch_command(self, command): if isinstance(command, list): cmd_fmt = ' '.join(command) else: cmd_fmt = command verbose(f"Docker host - run batch command '{cmd_fmt}'") if dryrun(): print(f"dry: (batch-in-container) {cmd_fmt}") else: socket = self.container.exec_run( cmd=command, stream=True, demux=True, workdir=self.config.project_dir_container) for (stdout, stderr) in socket.output: if stdout: sys.stdout.buffer.write(b'... ') sys.stdout.buffer.write(stdout) if stderr: sys.stderr.buffer.write(b'+++ ') sys.stderr.buffer.write(stderr)
def fetch(cls, force=None): project_dir = os.getcwd() for entry in cls.repos: source_path = os.path.join(project_dir, entry["source_dir"]) clone = False if os.path.isdir(source_path): if force: # remove first and clone later verbose("Remove old sources - i.e. forced update") shutil.rmtree(source_path) clone = True else: verbose(f"Update git repo: {entry['git_url']}") os.chdir(source_path) run_cmd(["git", "remote", "update"]) os.chdir(project_dir) else: clone = True if clone: verbose(f"Clone git repo: {entry['git_url']}") clone_path = os.path.abspath(os.path.join(source_path, "..")) os.makedirs(clone_path, exist_ok=True) os.chdir(clone_path) run_cmd(["git", "clone", entry["git_url"], source_path]) os.chdir(project_dir)
def __init__(cls): """ Initialize from Rontofile: Rontofile syntax is: git: - source_dir: sources/poky git_url: git://git.yoctoproject.org/poky """ model = get_model() if "git" in model: # skip totally if git is not set. is_command_available_or_exit(["git", "--version"]) if len(cls.repos) > 0: # is already initialized or git is not defined as fetcher return verbose(f"Config base: Git repositories") if isinstance(model["git"], list): for entry in model["git"]: verbose(f"Configured git repo: {entry['git_url']}") # we read directly since variable replacement does not # make sense for repository specification url = get_value(["git_url"], entry) source = get_value(["source_dir"], entry) if url and source: cls.repos.append(dict(git_url=url, source_dir=source)) # if ( # isinstance(entry, dict) # and "source_dir" in entry # and isinstance(entry["source_dir"], str) # and "git_url" in entry # and isinstance(entry["git_url"], str) #): # cls.repos.append(entry) if len(cls.repos) == 0: # initialize with poky default if nothing is given cls.repos.append({ "source_dir": "sources/poky", "git_url": "git://git.yoctoproject.org/poky", })