def run_interactive_build_command(self, command):
verbose(f"Docker host - run interactive command '{command}'")
if dryrun():
print(f"dry: (interactive-in-container) {command}")
else:
try:
socket = self.container.exec_run(
cmd=command,
tty=True,
stdin=True,
socket=True,
demux=False,
workdir=self.config.project_dir_container)
socket.output._sock.send(b'export PSADD="(c*r)"\n')
while True:
r, w, e = select.select([sys.stdin, socket.output._sock],
[],
[sys.stdin, socket.output._sock])
if sys.stdin in r:
d = os.read(sys.stdin.fileno(), 10240)
socket.output._sock.send(d)
elif socket.output._sock in r:
data = socket.output._sock.recv(16384)
os.write(sys.stdout.fileno(), data)
if sys.stdin in e or socket.output._sock in e:
break # leave the loop
except Exception as err:
verbose(f"Exception: {err}")
def __init__(self, targets_file=None):
verbose(f"Target Builder")
super().__init__()
self.targets = get_targets(targets_file)
self.do_packageindex = get_value_with_default(
["build", "packageindex"])
self.context = BatchContext(self.source_line)
def create_dir_and_dockerfile(
yocto_bitbaker_image="almedso/yocto-bitbaker:latest",
yocto_user_home="/home/yocto"):
"""
create a temporary directory and add a Dockerfile
to create a privatized container
"""
uid = os.getuid()
gid = os.getgid()
verbose(f"Inject uid {uid} and gid {gid}")
dockerfile = f"""
FROM {yocto_bitbaker_image}
RUN pip3 install --upgrade ronto
RUN groupadd --gid {gid} yocto || true && \
useradd --uid {uid} --gid {gid} --home {yocto_user_home} \
--create-home --shell /bin/bash yocto
USER yocto
"""
dir = tempfile.mkdtemp()
filename = os.path.join(dir, "Dockerfile")
with open(filename, "w") as f:
f.write(dockerfile)
return dir
def env_val(key, value):
env = os.getenv(key)
if env:
verbose(f"Read from environment {key}: {env}")
return env
else:
return value
def start_container(self):
if dryrun():
print(f"dry: Start container: {self.container_name}")
else:
verbose(f"Docker container status: {self.container.status}")
if self.container.status != "running":
verbose(f"Start docker container")
self.container.start()
def run_init():
"""
Source the init script once to place/ update build dir structure
i.e. create build dir, conf dir and add local.conf, bblayer.conf
"""
source_line = init_to_source_in()
verbose(f"Run init: {source_line[:-1]}")
run_cmd(["bash", "-c", source_line])
def init_process(args):
verbose("Process init command")
clean_init(rebuild_conf=args.rebuild_conf,
clean_conf_dir=args.clean_conf,
clean_build_dir=args.clean_build)
run_init()
siteconf = SiteConfigHandler()
siteconf.handle()
def __init__(self):
script = get_init_script()
self.build_dir = get_init_build_dir()
# There is no need to source something with templatedir
# this is done potentially in init command
source_line = "source " + script + " " + self.build_dir
verbose(f"Builder init sourcing: {source_line}")
self.source_line = source_line
def publish_target(target):
publish_dir = publish_image_dir()
if publish_dir != None:
# make sure directories are available
os.makedirs(publish_dir, exist_ok=True)
if 'publish' in target:
for image_machine in image_files(target):
verbose(f"Publish target {os.path.basename(image_machine)}")
run_cmd(['cp', '-fL', image_machine, publish_dir])
def check_version(version):
if version != None:
try:
verbose(f"Check rontofile version {version}")
int_version = int(version)
if int_version > 1:
raise VersionError()
except ValueError:
raise VersionError()
def build_process(args):
verbose("Process build command")
if args.interactive and not args.list_targets:
builder = InteractiveBuilder()
else:
builder = TargetBuilder()
if args.list_targets:
builder.list_targets()
else:
builder.build()
def fetch(cls, force=None):
if cls.url != "":
verbose(f"Init repo from {cls.url}")
run_cmd([
"repo", "init", "-u", cls.url, "-m", cls.manifest, "-b",
cls.branch
])
force_sync = "--force-sync" if force else ""
verbose(f"Sync repo {force_sync}")
run_cmd(["repo", "sync", force_sync])
def build(self):
for target in self.targets:
# print instead of verbose since bitbake is verbose anyway
print(f"****************************************************")
print(f"* Build {target['image']} for {target['machine']}")
print(f"****************************************************")
self.context.run_context(
f"MACHINE={target['machine']} bitbake {target['image']}")
if self.do_packageindex:
verbose("Do package index")
self.context.run_context("bitbake package-index")
def verify_target_specification(raw_targets):
verbose("Verify target specifications")
targets = []
if (raw_targets and isinstance(raw_targets, list)):
for target in raw_targets:
if (isinstance(target, dict) and "machine" in target
and isinstance(target["machine"], str)
and "image" in target
and isinstance(target["image"], str)):
targets.append(target)
return targets
def inject_command(self, command):
if command != '':
verbose(f"Run command: {command}")
composed_cmd = self.source_line + '\n'
self.process.stdin.write(composed_cmd.encode())
composed_cmd = command + '\n'
self.process.stdin.write(composed_cmd.encode())
composed_cmd = 'exit\n'
self.process.stdin.write(composed_cmd.encode())
composed_cmd = 'exit\n'
self.process.stdin.write(composed_cmd.encode())
def build_privatized_docker_image(self):
if dryrun():
print(f"dry: Build or get privatized docker image: " \
f"{self.config.privatized_image}")
else:
try:
image_label = self.config.privatized_image + ":latest"
image = self.docker.images.get(image_label)
verbose(f"Privatized image {image_label} exists" \
" - no need to build")
except docker.errors.ImageNotFound as _err:
self._build_privatized_docker_image()
def update_defaults():
"""
Section defaults to deal with, like:
defaults:
FOO: 'foo'
BAR: 'bar'
"""
global variables_
verbose("Update default variables")
if "defaults" in model_ and isinstance(model_["defaults"], dict):
for default in model_["defaults"]:
variables_[default] = env_val(default, model_["defaults"][default])
def _build_privatized_docker_image(self):
verbose("Build privatized docker image")
privatized_docker_image = self.config.get_privatized_image()
yocto_docker_image = self.config.get_image()
dir = create_dir_and_dockerfile(yocto_docker_image,
self.yocto_user_home)
if dryrun():
with open(os.path.join(dir, "Dockerfile"), "r") as f:
print("dry: privatizing Dockerfile" + f.read())
run_cmd(["docker", "build", "-t", privatized_docker_image, dir])
os.remove(os.path.join(dir, "Dockerfile")) # cleanup Dockerfile
os.rmdir(dir) # cleanup temporary directory
def get_targets(targets_file):
""" Get list of targets plus inspection of them """
raw_targets = get_targets_from_yaml_file(targets_file)
if not raw_targets:
verbose("Check for targets directly defined in 'ronto.yml'")
raw_targets = get_value(["build", "targets"])
targets = verify_target_specification(raw_targets)
if len(targets) == 0:
verbose(" No verified target found -> use default target")
# Add a default machine/image combination as of yocto docu
# getting started section.
targets.append({"machine": "qemux86", "image": "core-image-sato"})
return targets
def run_context(self, command):
if dryrun():
print(f"dry - Run build command: {command}")
else:
try:
self.process = subprocess.Popen(
"bash",
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
verbose(f"Start Bash session: Pid {self.process.pid}")
self.inject_command(command)
self.process.communicate()
except subprocess.SubprocessError:
pass
verbose(f"Stop bash session: Pid {self.process.pid}")
def run_context(self):
# carridge return is required due to change terminal discipline
verbose(f"run context\r")
while self.process.poll() is None:
r, w, e = select.select([sys.stdin, self.master_fd], [],
[sys.stdin, self.master_fd])
if sys.stdin in r:
d = os.read(sys.stdin.fileno(), 10240)
os.write(self.master_fd, d)
elif self.master_fd in r:
o = os.read(self.master_fd, 10240)
if o:
os.write(sys.stdout.fileno(), o)
if sys.stdin in e or self.master_fd in e:
break
termios.tcsetattr(sys.stdin, termios.TCSADRAIN, self.old_tty)
verbose(f"Stop bash session: Pid {self.process.pid}")
def process(args):
docker = docker_factory()
# only on docker host successful/usefull
if docker:
docker.build_privatized_docker_image()
docker.create_container()
docker.start_container()
docker.run_command(args.cmd, args.interactive)
docker.stop_container()
if args.rm_container:
docker.remove_container()
if args.rm_priv_image:
docker.remove_privatized_image()
if args.rm_all:
docker.remove_all()
else:
verbose("No docker environment")
def handle(self):
"""
Update site.conf if overwrite and file available
Create site.conf if source file available
"""
dest_config_dir = os.path.join(os.getcwd(), self.build_dir, "conf")
dest_site_conf_file = os.path.join(dest_config_dir, "site.conf")
if not os.path.isfile(dest_site_conf_file):
# site.conf file does not exit (Create scenario)
src_site_conf_file = os.path.join(os.getcwd(), self.sitefile)
if os.path.isfile(src_site_conf_file):
verbose(f"Create site.conf from: {src_site_conf_file}")
if dryrun():
print(
f"copy {src_site_conf_file} to {dest_site_conf_file}")
else:
os.makedirs(dest_config_dir, exist_ok=True)
copyfile(src_site_conf_file, dest_site_conf_file)
def run_command(self, command, interactive_flag=False):
# cleanup the ronto command path
if isinstance(command, list):
if 'ronto' in command[0]:
command[0] = 'ronto' # get rid of of host local path
if isinstance(command, str):
p = re.compile('^([\w/]*ronto)')
command = p.sub('ronto', command)
if interactive_flag \
or '-i' in command \
or '--interactive' in command: # in works on lists and on strings
self.run_interactive_build_command(command)
else:
self.run_batch_command(command)
if isinstance(command, list):
command = ' '.join(command)
verbose(f"Docker command '{command}' finished - returned to host")
def get_targets_from_yaml_file(targets_file):
"""
Read targets from file (either input or defined in ronto.yml )
@target_file: relative path of targets file from project directory
or None
@returns None of structure of the file
"""
if not targets_file:
targets_file = get_value_with_default(['build', 'targets_file'])
if targets_file:
verbose(f"Read targets from file: {targets_file}")
try:
with open(targets_file) as file:
return yaml.load(file, Loader=yaml.FullLoader)
except FileNotFoundError:
print(f"File with target specifications not found -> fall back",
file=sys.stderr)
return None
return None
def __init__(cls):
if exists(["docker"]):
verbose("Docker configuration found")
cls._use_docker = True
cls.image = get_value_with_default(["docker", "image"],
"almedso/yocto-bitbaker:latest")
cls.privatized_image = get_value_with_default(
["docker", "privatized_image"], "my-yocto-bitbaker")
cls.project_dir_container = get_value_with_default(
["docker", "project_dir"], "/yocto/root")
cls.cache_dir_host = get_value_with_default(
["docker", "cache_dir", "host"],
os.path.abspath(os.path.join(PROJECT_DIR_HOST, '..', 'cache')))
cls.cache_dir_container = get_value_with_default(
["docker", "cache_dir", "container"], "/yocto/cache")
cls.publish_dir_host = get_value_with_default(
["docker", "publish_dir", "host"], "")
cls.publish_dir_container = get_value_with_default(
["docker", "publish_dir", "container"], "/yocto/publish")
def __init__(cls):
"""
Initialize from Rontofile:
Rontofile syntax is:
repo:
url: git://host/git-manifest-repo.git
manifest: release-xyz.xml
branch: master
"""
if 'repo' in get_model():
# skip totally if repo is not set.
is_command_available_or_exit(["repo", "--version"])
verbose(f"Config base: Google manifest repository")
cls.url = get_value(['repo', 'url'])
if cls.url == None:
print("repo URL cannot be determined", file=sys.stderr)
sys.exit(1)
cls.branch = get_value_with_default(["repo", "branch"], 'master')
cls.manifest = get_value_with_default(["repo", "manifest"],
'default.xml')
def run_batch_command(self, command):
if isinstance(command, list):
cmd_fmt = ' '.join(command)
else:
cmd_fmt = command
verbose(f"Docker host - run batch command '{cmd_fmt}'")
if dryrun():
print(f"dry: (batch-in-container) {cmd_fmt}")
else:
socket = self.container.exec_run(
cmd=command,
stream=True,
demux=True,
workdir=self.config.project_dir_container)
for (stdout, stderr) in socket.output:
if stdout:
sys.stdout.buffer.write(b'... ')
sys.stdout.buffer.write(stdout)
if stderr:
sys.stderr.buffer.write(b'+++ ')
sys.stderr.buffer.write(stderr)
def fetch(cls, force=None):
project_dir = os.getcwd()
for entry in cls.repos:
source_path = os.path.join(project_dir, entry["source_dir"])
clone = False
if os.path.isdir(source_path):
if force:
# remove first and clone later
verbose("Remove old sources - i.e. forced update")
shutil.rmtree(source_path)
clone = True
else:
verbose(f"Update git repo: {entry['git_url']}")
os.chdir(source_path)
run_cmd(["git", "remote", "update"])
os.chdir(project_dir)
else:
clone = True
if clone:
verbose(f"Clone git repo: {entry['git_url']}")
clone_path = os.path.abspath(os.path.join(source_path, ".."))
os.makedirs(clone_path, exist_ok=True)
os.chdir(clone_path)
run_cmd(["git", "clone", entry["git_url"], source_path])
os.chdir(project_dir)
def __init__(cls):
"""
Initialize from Rontofile:
Rontofile syntax is:
git:
- source_dir: sources/poky
git_url: git://git.yoctoproject.org/poky
"""
model = get_model()
if "git" in model:
# skip totally if git is not set.
is_command_available_or_exit(["git", "--version"])
if len(cls.repos) > 0:
# is already initialized or git is not defined as fetcher
return
verbose(f"Config base: Git repositories")
if isinstance(model["git"], list):
for entry in model["git"]:
verbose(f"Configured git repo: {entry['git_url']}")
# we read directly since variable replacement does not
# make sense for repository specification
url = get_value(["git_url"], entry)
source = get_value(["source_dir"], entry)
if url and source:
cls.repos.append(dict(git_url=url, source_dir=source))
# if (
# isinstance(entry, dict)
# and "source_dir" in entry
# and isinstance(entry["source_dir"], str)
# and "git_url" in entry
# and isinstance(entry["git_url"], str)
#):
# cls.repos.append(entry)
if len(cls.repos) == 0:
# initialize with poky default if nothing is given
cls.repos.append({
"source_dir": "sources/poky",
"git_url": "git://git.yoctoproject.org/poky",
})
