def retrieve_apiversions(libfullpath):
"""Retrieve version definitions from .so library
using readelf binary
Args:
libfullpath (string): fullpath to .so library
Returns:
array[String]: version definitions
"""
if not os.path.exists(libfullpath):
return []
apiversions = []
command = f"readelf -V {libfullpath}"
## logger.debug(f"Executing command: {command}")
success, output = Utils.run_process_and_return_output(command)
if success == 0:
## parses something like below to get out 'GLIBC_2.6' :
## 0x005c: Rev: 1 Flags: none Index: 4 Cnt: 2 Name: GLIBC_2.6
in_version_definition_section = False
for line in output.splitlines():
if (in_version_definition_section):
if ("Version needs section" in line):
in_version_definition_section = False
TAG = "Name: "
idx = line.find(TAG)
if (line.startswith(" 0x") and (idx > 0)):
apiversions.append(line[idx+len(TAG):])
elif ("Version definition section" in line):
in_version_definition_section = True
return apiversions
def unpack_image(self, tag, delete=False):
"""Attempt to unpack an OCI image in a given directory to an OCI bundle
without any modifications
Calls out to umoci to handle the base conversion
"""
# If umoci isn't installed, can't unpack
if not self.umoci_found:
logger.error("Cannot unpack image as cannot find umoci")
return
umoci_command = f'umoci unpack --rootless --image {self.src}:{tag} {self.dest}'
logger.debug(umoci_command)
success = Utils().run_process(umoci_command)
if success == 0:
logger.info(f"Unpacked image successfully to {self.dest}")
if delete:
logger.debug("Deleting downloaded image")
shutil.rmtree(self.src)
return True
else:
logger.warning("Umoci failed to unpack the image")
return False
def download_image(self, url, creds, platform_cfg):
"""Attempt to download the specified image using skopeo
Will download and extract the image to /tmp
Args:
url (string): URL to download the image from (e.g. docker://hello-world:latest)
creds (string): Credentials for the OCI registry in the form username:password
platform_cfg (dict): Platform template
Returns:
string: Path to downloaded image
"""
# If skopeo isn't installed, can't download
if not self.skopeo_found:
logger.error("Cannot download image as cannot find skopeo")
return
image_tag = self.get_image_tag(url)
# Get arch from config
if 'arch' not in platform_cfg:
logger.error("Platform architecture is not defined", err=True)
return ""
else:
arch = platform_cfg['arch'].get('arch')
variant = platform_cfg['arch'].get('variant')
if 'os' not in platform_cfg:
logger.error("Platform OS is not defined", err=True)
return ""
else:
platform_os = platform_cfg.get('os')
# Save the image to a temp dir. Use uuid to generate a unique name
if not os.path.exists('/tmp/bundlegen'):
os.makedirs('/tmp/bundlegen')
now = time.strftime("%Y%m%d-%H%M%S")
destination = f'/tmp/bundlegen/{now}_{Utils.get_random_string()}'
logger.info(f"Downloading image to {destination}...")
# Build the command to skopeo
skopeo_command = f'skopeo '
if (os.path.exists(
os.path.expanduser('~/.config/containers/policy.json'))
or os.path.exists('/etc/containers/policy.json')):
logger.debug('Found a policy.json file for skopeo')
else:
logger.debug(
'Did not find a policy.json file for skopeo. Will use insecure-policy flag for skopeo!'
)
skopeo_command += '--insecure-policy '
if creds:
skopeo_command += f'--src-creds {creds} '
if variant:
skopeo_command += f'--override-os {platform_os} --override-arch {arch} --override-variant {variant} '
else:
skopeo_command += f'--override-os {platform_os} --override-arch {arch} '
skopeo_command += f'copy {url} oci:{destination}:{image_tag}'
logger.debug(skopeo_command)
# Run skopeo, and stream the output to the console
success = Utils.run_process(skopeo_command)
if success == 0:
logger.success(
f"Downloaded image from {url} successfully to {destination}")
return destination
else:
logger.warning("Skopeo failed to download the image")
return None
def generate_bundle(options: message.Message) -> Tuple[Result, str]:
"""
Actually do the work to generate the bundle
"""
# Create a random dir to save the generated bundle, before copying to output directory
outputdir = os.path.abspath(
os.path.join(os.environ.get('BUNDLEGEN_TMP_DIR'),
Utils.get_random_string(5)))
selected_platform = STBPlatform(
options.platform, options.searchpath
or os.environ.get('RDK_PLATFORM_SEARCHPATH'))
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {options.platform}")
return (Result.FATAL_ERROR, "")
if not options.image_url:
logger.error("Image URL is not set - cannot generate bundle")
return (Result.FATAL_ERROR, "")
logger.success(
f"Starting Bundle Generation from image '{options.image_url}' for platform {options.platform} (UUID: {options.uuid})"
)
creds = os.environ.get("RDK_OCI_REGISTRY_CREDS")
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(options.image_url, creds,
selected_platform.get_config())
if not img_path:
logger.error("Failed to download image")
return (Result.TRANSIENT_ERROR, "")
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(options.image_url)
img_unpacker = ImageUnpackager(img_path, outputdir)
img_unpacker.unpack_image(tag, delete=True)
# Load app metadata
metadata_from_image = img_unpacker.get_app_metadata_from_img()
custom_app_metadata = options.app_metadata
app_metadata_dict = {}
if not metadata_from_image and not custom_app_metadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen"
)
return (Result.FATAL_ERROR, "")
if not metadata_from_image and custom_app_metadata:
# Use custom metadata
app_metadata_dict = custom_app_metadata
elif metadata_from_image and custom_app_metadata:
logger.warning(
"Image contains app metadata and custom metadata provided. Using custom metadata"
)
app_metadata_dict = custom_app_metadata
img_unpacker.delete_img_app_metadata()
else:
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
if options.app_id:
app_metadata_dict['id'] = options.app_id
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(
selected_platform.get_config(),
outputdir,
app_metadata_dict,
False,
options.lib_match_mode.value,
options.createmountpoints,
)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
logger.error(
f"App is not compatible with platform {options.platform}, cannot generate bundle"
)
return (Result.FATAL_ERROR, "")
success = processor.begin_processing()
if not success:
logger.error("Failed to generate bundle")
# This might have been some weird issue, so re-queue to try again later
return (Result.TRANSIENT_ERROR, "")
if options.output_filename:
tarball_name = options.output_filename
else:
tarball_name = app_metadata_dict["id"] + Utils.get_random_string(6)
tmp_path = os.path.join(os.environ.get('BUNDLEGEN_TMP_DIR'),
f"{tarball_name}.tar.gz")
persistent_path = os.path.join(
options.outputdir or os.environ.get('BUNDLE_STORE_DIR'),
f"{tarball_name}.tar.gz")
tarball_settings = processor.platform_cfg.get('tarball')
file_ownership_user = tarball_settings.get(
'fileOwnershipSameAsUser') if tarball_settings else None
file_mask = tarball_settings.get('fileMask') if tarball_settings else None
container_uid_gid = processor.get_real_uid_gid()
uid = container_uid_gid[
0] if container_uid_gid[0] and file_ownership_user else None
gid = container_uid_gid[
1] if container_uid_gid[1] and file_ownership_user else None
Utils.create_tgz(outputdir, tmp_path, uid, gid, file_mask)
# Move to persistent storage
logger.debug(
f"Moving '{tmp_path}' to {options.outputdir or os.environ.get('BUNDLE_STORE_DIR')}"
)
shutil.move(tmp_path, options.outputdir
or os.environ.get('BUNDLE_STORE_DIR'))
return (Result.SUCCESS, persistent_path)
def index():
get_templates()
form = GenerateForm()
form.platform.choices = get_templates()
if request.method == 'GET':
return render_template('index.html', form=form)
if form.validate_on_submit():
print("Valid form submission")
# Do BundleGen work
outputdir = os.path.abspath(
os.path.join(TMP_DIR, Utils.get_random_string(5)))
selected_platform = STBPlatform(form.platform.data)
if not selected_platform.found_config():
print(f"Could not find config for platform {form.platform.data}")
raise AppError("Could not find platform")
img_url = ""
creds = ""
if form.image_url.data:
# If downloading from URL, just use that as-is
img_url = form.image_url.data
# Add creds if given
if form.registry_uname.data and form.registry_password.data:
creds = f"{form.registry_uname.data}:{form.registry_password.data}"
elif form.uploaded_img.data:
# Got an uploaded image (hopefully a tar.gz!)
f = form.uploaded_img.data
filename = secure_filename(f.filename)
upload_filepath = os.path.join(UPLOAD_FOLDER, filename)
f.save(upload_filepath)
# Extract tar
# Extract the .tar to a temp directory
img_temp_path = tempfile.mkdtemp()
with tarfile.open(upload_filepath) as tar:
tar.extractall(img_temp_path)
# Delete tar
os.remove(upload_filepath)
img_url = f"oci:{img_temp_path}:latest"
creds = None
if not img_url:
print("IMG URL is empty")
raise AppError("Image URL cannot be empty")
# Download Image
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(
img_url, creds, selected_platform.get_config())
if not img_path:
logger.error("Failed to download image")
raise AppError("Image download failed")
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(img_url)
img_unpacker = ImageUnpackager(img_path, outputdir)
img_unpacker.unpack_image(tag, delete=True)
# Load app metadata
metadata_from_image = img_unpacker.get_app_metadata_from_img()
custom_app_metadata = form.app_metadata.data
app_metadata_dict = {}
if not metadata_from_image and not custom_app_metadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen"
)
raise AppError("No Metadata provided")
if not metadata_from_image and custom_app_metadata:
# Use custom metadata
app_metadata_dict = json.loads(custom_app_metadata)
elif metadata_from_image and custom_app_metadata:
logger.warning(
"Image contains app metadata and custom metadata provided. Using custom metadata"
)
app_metadata_dict = json.loads(custom_app_metadata)
img_unpacker.delete_img_app_metadata()
else:
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(selected_platform.get_config(), outputdir,
app_metadata_dict, False,
form.lib_match.data)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
raise AppError("App incompatible")
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
raise AppError("Something went wrong")
tarball_name = app_metadata_dict["id"] + Utils.get_random_string(6)
Utils.create_tgz(outputdir, tarball_name)
logger.success(
f"Successfully generated bundle at {tarball_name}.tar.gz")
# Move to persistant storage
print(f"Moving '{tarball_name}.tar.gz' to {BUNDLE_STORE_DIR}")
shutil.move(f'{tarball_name}.tar.gz', BUNDLE_STORE_DIR)
return jsonify(success=True)
def generate(image, outputdir, platform, appmetadata, searchpath, creds, yes):
"""Generate an OCI Bundle for a specified platform
"""
logger.info(f'Generating new OCI bundle* from {image} for {platform}')
outputdir = os.path.abspath(outputdir)
# Check if the output dir already exists
if os.path.exists(outputdir):
if not yes:
click.confirm(
f"The directory {outputdir} already exists. Are you sure you want to continue? The contents of this directory will be deleted",
abort=True)
# Delete existing directory
shutil.rmtree(outputdir)
# Load the config for the platform
selected_platform = STBPlatform(platform, searchpath)
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {platform}")
return
# Get the app metadata as a dictionary
# TODO:: Metadata will be embedded in the image moving forward
app_metadata_dict = {}
if os.path.exists(appmetadata):
with open(appmetadata) as metadata:
app_metadata_dict = json.load(metadata)
else:
logger.error(f"Cannot find app metadata file {appmetadata}")
return
# Download the image to a temp directory
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(image, creds,
selected_platform.get_config())
if not img_path:
return
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(image)
img_unpacker = ImageUnpackager()
unpack_success = img_unpacker.unpack_image(img_path, tag, outputdir)
if not unpack_success:
return
# Delete the downloaded image now we've unpacked it
logger.info(f"Deleting {img_path}")
shutil.rmtree(img_path)
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(selected_platform.get_config(), outputdir,
app_metadata_dict)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
return
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
return
# Processing finished, now create a tarball of the output directory
Utils.create_tgz(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.tar.gz")
def generate(image, outputdir, platform, searchpath, creds, ipk, appmetadata,
yes, nodepwalking, libmatchingmode):
"""Generate an OCI Bundle for a specified platform
"""
logger.info(f'Generating new OCI bundle* from {image} for {platform}')
outputdir = os.path.abspath(outputdir)
# Check if the output dir already exists
if os.path.exists(outputdir):
if not yes:
click.confirm(
f"The directory {outputdir} already exists. Are you sure you want to continue? The contents of this directory will be deleted",
abort=True)
# Delete existing directory
shutil.rmtree(outputdir)
# Load the config for the platform
selected_platform = STBPlatform(platform, searchpath)
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {platform}")
return
# Download the image to a temp directory
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(image, creds,
selected_platform.get_config())
if not img_path:
return
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(image)
img_unpacker = ImageUnpackager()
unpack_success = img_unpacker.unpack_image(img_path, tag, outputdir)
if not unpack_success:
return
# Delete the downloaded image now we've unpacked it
logger.info(f"Deleting {img_path}")
shutil.rmtree(img_path)
# Load app metadata
app_metadata_file = ""
app_metadata_image_path = os.path.join(outputdir, "rootfs",
"appmetadata.json")
image_metadata_exists = os.path.exists(app_metadata_image_path)
if not image_metadata_exists and not appmetadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen"
)
return
elif not image_metadata_exists and appmetadata:
# No metadata in image, but custom file provided
if not os.path.exists(appmetadata):
logger.error(f'App metadata file {appmetadata} does not exist')
return
app_metadata_file = appmetadata
elif image_metadata_exists and appmetadata:
# Got two options for metadata, which one do we want?
if click.confirm(
"Metadata found in image, but custom metadata provided. Use custom metadata?"
):
app_metadata_file = appmetadata
else:
app_metadata_file = app_metadata_image_path
else:
app_metadata_file = app_metadata_image_path
logger.debug(f"Loading metadata from {app_metadata_file}")
# Load the metadata
app_metadata_dict = {}
with open(app_metadata_file) as metadata:
app_metadata_dict = json.load(metadata)
# remove app metadata from image rootfs
if image_metadata_exists:
os.remove(app_metadata_image_path)
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(selected_platform.get_config(), outputdir,
app_metadata_dict, nodepwalking,
libmatchingmode)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
return
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
return
# Processing finished, now create a tarball/ipk of the output directory
if ipk:
# create control file
Utils.create_control_file(selected_platform.get_config(),
app_metadata_dict)
Utils.create_ipk(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.ipk")
else:
Utils.create_tgz(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.tar.gz")
def generate(image, outputdir, platform, searchpath, creds, ipk, appmetadata, yes, nodepwalking, libmatchingmode, createmountpoints, appid):
"""Generate an OCI Bundle for a specified platform
"""
logger.info(f'Generating new OCI bundle* from {image} for {platform}')
outputdir = os.path.abspath(outputdir)
# Check if the output dir already exists
if os.path.exists(outputdir):
if not yes:
click.confirm(
f"The directory {outputdir} already exists. Are you sure you want to continue? The contents of this directory will be deleted", abort=True)
# Delete existing directory
shutil.rmtree(outputdir)
# Load the config for the platform
selected_platform = STBPlatform(platform, searchpath)
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {platform}")
sys.exit(1)
# Download the image to a temp directory
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(
image, creds, selected_platform.get_config())
if not img_path:
sys.exit(1)
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(image)
img_unpacker = ImageUnpackager(src=img_path, dst=outputdir)
unpack_success = img_unpacker.unpack_image(tag, delete=True)
if not unpack_success:
sys.exit(1)
# Load app metadata
metadata_from_image = img_unpacker.get_app_metadata_from_img()
appmetadata = os.path.abspath(appmetadata) if appmetadata else None
app_metadata_dict = {}
if not metadata_from_image and not appmetadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen")
sys.exit(1)
if not metadata_from_image and appmetadata:
# No metadata in image, but custom file provided
if not os.path.exists(appmetadata):
logger.error(f'App metadata file {appmetadata} does not exist')
sys.exit(1)
with open(appmetadata) as metadata:
logger.debug(f"Loading metadata from {appmetadata}")
app_metadata_dict = json.load(metadata)
elif metadata_from_image and appmetadata:
# Got two options for metadata, which one do we want?
if click.confirm("Metadata found in image, but custom metadata provided. Use custom metadata?"):
with open(appmetadata) as metadata:
logger.debug(f"Loading metadata from {appmetadata}")
app_metadata_dict = json.load(metadata)
else:
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
else:
# Take metadata from image
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
if appid:
app_metadata_dict['id'] = appid
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(
selected_platform.get_config(), outputdir, app_metadata_dict, nodepwalking, libmatchingmode, createmountpoints)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
sys.exit(2)
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
sys.exit(3)
# Processing finished, now create a tarball/ipk of the output directory
if ipk:
# create control file
Utils.create_control_file(
selected_platform.get_config(), app_metadata_dict)
Utils.create_ipk(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.ipk")
else:
tarball_settings = processor.platform_cfg.get('tarball')
file_ownership_user = tarball_settings.get('fileOwnershipSameAsUser') if tarball_settings else None
file_mask = tarball_settings.get('fileMask') if tarball_settings else None
container_uid_gid = processor.get_real_uid_gid()
uid = container_uid_gid[0] if container_uid_gid[0] and file_ownership_user else None
gid = container_uid_gid[1] if container_uid_gid[1] and file_ownership_user else None
Utils.create_tgz(outputdir, outputdir, uid, gid, file_mask)
logger.success(f"Successfully generated bundle at {outputdir}.tar.gz")