def generate_bundle(options: message.Message) -> Tuple[Result, str]:
"""
Actually do the work to generate the bundle
"""
# Create a random dir to save the generated bundle, before copying to output directory
outputdir = os.path.abspath(
os.path.join(os.environ.get('BUNDLEGEN_TMP_DIR'),
Utils.get_random_string(5)))
selected_platform = STBPlatform(
options.platform, options.searchpath
or os.environ.get('RDK_PLATFORM_SEARCHPATH'))
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {options.platform}")
return (Result.FATAL_ERROR, "")
if not options.image_url:
logger.error("Image URL is not set - cannot generate bundle")
return (Result.FATAL_ERROR, "")
logger.success(
f"Starting Bundle Generation from image '{options.image_url}' for platform {options.platform} (UUID: {options.uuid})"
)
creds = os.environ.get("RDK_OCI_REGISTRY_CREDS")
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(options.image_url, creds,
selected_platform.get_config())
if not img_path:
logger.error("Failed to download image")
return (Result.TRANSIENT_ERROR, "")
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(options.image_url)
img_unpacker = ImageUnpackager(img_path, outputdir)
img_unpacker.unpack_image(tag, delete=True)
# Load app metadata
metadata_from_image = img_unpacker.get_app_metadata_from_img()
custom_app_metadata = options.app_metadata
app_metadata_dict = {}
if not metadata_from_image and not custom_app_metadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen"
)
return (Result.FATAL_ERROR, "")
if not metadata_from_image and custom_app_metadata:
# Use custom metadata
app_metadata_dict = custom_app_metadata
elif metadata_from_image and custom_app_metadata:
logger.warning(
"Image contains app metadata and custom metadata provided. Using custom metadata"
)
app_metadata_dict = custom_app_metadata
img_unpacker.delete_img_app_metadata()
else:
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
if options.app_id:
app_metadata_dict['id'] = options.app_id
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(
selected_platform.get_config(),
outputdir,
app_metadata_dict,
False,
options.lib_match_mode.value,
options.createmountpoints,
)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
logger.error(
f"App is not compatible with platform {options.platform}, cannot generate bundle"
)
return (Result.FATAL_ERROR, "")
success = processor.begin_processing()
if not success:
logger.error("Failed to generate bundle")
# This might have been some weird issue, so re-queue to try again later
return (Result.TRANSIENT_ERROR, "")
if options.output_filename:
tarball_name = options.output_filename
else:
tarball_name = app_metadata_dict["id"] + Utils.get_random_string(6)
tmp_path = os.path.join(os.environ.get('BUNDLEGEN_TMP_DIR'),
f"{tarball_name}.tar.gz")
persistent_path = os.path.join(
options.outputdir or os.environ.get('BUNDLE_STORE_DIR'),
f"{tarball_name}.tar.gz")
tarball_settings = processor.platform_cfg.get('tarball')
file_ownership_user = tarball_settings.get(
'fileOwnershipSameAsUser') if tarball_settings else None
file_mask = tarball_settings.get('fileMask') if tarball_settings else None
container_uid_gid = processor.get_real_uid_gid()
uid = container_uid_gid[
0] if container_uid_gid[0] and file_ownership_user else None
gid = container_uid_gid[
1] if container_uid_gid[1] and file_ownership_user else None
Utils.create_tgz(outputdir, tmp_path, uid, gid, file_mask)
# Move to persistent storage
logger.debug(
f"Moving '{tmp_path}' to {options.outputdir or os.environ.get('BUNDLE_STORE_DIR')}"
)
shutil.move(tmp_path, options.outputdir
or os.environ.get('BUNDLE_STORE_DIR'))
return (Result.SUCCESS, persistent_path)
def index():
get_templates()
form = GenerateForm()
form.platform.choices = get_templates()
if request.method == 'GET':
return render_template('index.html', form=form)
if form.validate_on_submit():
print("Valid form submission")
# Do BundleGen work
outputdir = os.path.abspath(
os.path.join(TMP_DIR, Utils.get_random_string(5)))
selected_platform = STBPlatform(form.platform.data)
if not selected_platform.found_config():
print(f"Could not find config for platform {form.platform.data}")
raise AppError("Could not find platform")
img_url = ""
creds = ""
if form.image_url.data:
# If downloading from URL, just use that as-is
img_url = form.image_url.data
# Add creds if given
if form.registry_uname.data and form.registry_password.data:
creds = f"{form.registry_uname.data}:{form.registry_password.data}"
elif form.uploaded_img.data:
# Got an uploaded image (hopefully a tar.gz!)
f = form.uploaded_img.data
filename = secure_filename(f.filename)
upload_filepath = os.path.join(UPLOAD_FOLDER, filename)
f.save(upload_filepath)
# Extract tar
# Extract the .tar to a temp directory
img_temp_path = tempfile.mkdtemp()
with tarfile.open(upload_filepath) as tar:
tar.extractall(img_temp_path)
# Delete tar
os.remove(upload_filepath)
img_url = f"oci:{img_temp_path}:latest"
creds = None
if not img_url:
print("IMG URL is empty")
raise AppError("Image URL cannot be empty")
# Download Image
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(
img_url, creds, selected_platform.get_config())
if not img_path:
logger.error("Failed to download image")
raise AppError("Image download failed")
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(img_url)
img_unpacker = ImageUnpackager(img_path, outputdir)
img_unpacker.unpack_image(tag, delete=True)
# Load app metadata
metadata_from_image = img_unpacker.get_app_metadata_from_img()
custom_app_metadata = form.app_metadata.data
app_metadata_dict = {}
if not metadata_from_image and not custom_app_metadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen"
)
raise AppError("No Metadata provided")
if not metadata_from_image and custom_app_metadata:
# Use custom metadata
app_metadata_dict = json.loads(custom_app_metadata)
elif metadata_from_image and custom_app_metadata:
logger.warning(
"Image contains app metadata and custom metadata provided. Using custom metadata"
)
app_metadata_dict = json.loads(custom_app_metadata)
img_unpacker.delete_img_app_metadata()
else:
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(selected_platform.get_config(), outputdir,
app_metadata_dict, False,
form.lib_match.data)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
raise AppError("App incompatible")
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
raise AppError("Something went wrong")
tarball_name = app_metadata_dict["id"] + Utils.get_random_string(6)
Utils.create_tgz(outputdir, tarball_name)
logger.success(
f"Successfully generated bundle at {tarball_name}.tar.gz")
# Move to persistant storage
print(f"Moving '{tarball_name}.tar.gz' to {BUNDLE_STORE_DIR}")
shutil.move(f'{tarball_name}.tar.gz', BUNDLE_STORE_DIR)
return jsonify(success=True)
def generate(image, outputdir, platform, searchpath, creds, ipk, appmetadata,
yes, nodepwalking, libmatchingmode):
"""Generate an OCI Bundle for a specified platform
"""
logger.info(f'Generating new OCI bundle* from {image} for {platform}')
outputdir = os.path.abspath(outputdir)
# Check if the output dir already exists
if os.path.exists(outputdir):
if not yes:
click.confirm(
f"The directory {outputdir} already exists. Are you sure you want to continue? The contents of this directory will be deleted",
abort=True)
# Delete existing directory
shutil.rmtree(outputdir)
# Load the config for the platform
selected_platform = STBPlatform(platform, searchpath)
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {platform}")
return
# Download the image to a temp directory
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(image, creds,
selected_platform.get_config())
if not img_path:
return
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(image)
img_unpacker = ImageUnpackager()
unpack_success = img_unpacker.unpack_image(img_path, tag, outputdir)
if not unpack_success:
return
# Delete the downloaded image now we've unpacked it
logger.info(f"Deleting {img_path}")
shutil.rmtree(img_path)
# Load app metadata
app_metadata_file = ""
app_metadata_image_path = os.path.join(outputdir, "rootfs",
"appmetadata.json")
image_metadata_exists = os.path.exists(app_metadata_image_path)
if not image_metadata_exists and not appmetadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen"
)
return
elif not image_metadata_exists and appmetadata:
# No metadata in image, but custom file provided
if not os.path.exists(appmetadata):
logger.error(f'App metadata file {appmetadata} does not exist')
return
app_metadata_file = appmetadata
elif image_metadata_exists and appmetadata:
# Got two options for metadata, which one do we want?
if click.confirm(
"Metadata found in image, but custom metadata provided. Use custom metadata?"
):
app_metadata_file = appmetadata
else:
app_metadata_file = app_metadata_image_path
else:
app_metadata_file = app_metadata_image_path
logger.debug(f"Loading metadata from {app_metadata_file}")
# Load the metadata
app_metadata_dict = {}
with open(app_metadata_file) as metadata:
app_metadata_dict = json.load(metadata)
# remove app metadata from image rootfs
if image_metadata_exists:
os.remove(app_metadata_image_path)
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(selected_platform.get_config(), outputdir,
app_metadata_dict, nodepwalking,
libmatchingmode)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
return
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
return
# Processing finished, now create a tarball/ipk of the output directory
if ipk:
# create control file
Utils.create_control_file(selected_platform.get_config(),
app_metadata_dict)
Utils.create_ipk(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.ipk")
else:
Utils.create_tgz(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.tar.gz")
def generate(image, outputdir, platform, appmetadata, searchpath, creds, yes):
"""Generate an OCI Bundle for a specified platform
"""
logger.info(f'Generating new OCI bundle* from {image} for {platform}')
outputdir = os.path.abspath(outputdir)
# Check if the output dir already exists
if os.path.exists(outputdir):
if not yes:
click.confirm(
f"The directory {outputdir} already exists. Are you sure you want to continue? The contents of this directory will be deleted",
abort=True)
# Delete existing directory
shutil.rmtree(outputdir)
# Load the config for the platform
selected_platform = STBPlatform(platform, searchpath)
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {platform}")
return
# Get the app metadata as a dictionary
# TODO:: Metadata will be embedded in the image moving forward
app_metadata_dict = {}
if os.path.exists(appmetadata):
with open(appmetadata) as metadata:
app_metadata_dict = json.load(metadata)
else:
logger.error(f"Cannot find app metadata file {appmetadata}")
return
# Download the image to a temp directory
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(image, creds,
selected_platform.get_config())
if not img_path:
return
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(image)
img_unpacker = ImageUnpackager()
unpack_success = img_unpacker.unpack_image(img_path, tag, outputdir)
if not unpack_success:
return
# Delete the downloaded image now we've unpacked it
logger.info(f"Deleting {img_path}")
shutil.rmtree(img_path)
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(selected_platform.get_config(), outputdir,
app_metadata_dict)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
return
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
return
# Processing finished, now create a tarball of the output directory
Utils.create_tgz(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.tar.gz")
def generate(image, outputdir, platform, searchpath, creds, ipk, appmetadata, yes, nodepwalking, libmatchingmode, createmountpoints, appid):
"""Generate an OCI Bundle for a specified platform
"""
logger.info(f'Generating new OCI bundle* from {image} for {platform}')
outputdir = os.path.abspath(outputdir)
# Check if the output dir already exists
if os.path.exists(outputdir):
if not yes:
click.confirm(
f"The directory {outputdir} already exists. Are you sure you want to continue? The contents of this directory will be deleted", abort=True)
# Delete existing directory
shutil.rmtree(outputdir)
# Load the config for the platform
selected_platform = STBPlatform(platform, searchpath)
if not selected_platform.found_config():
logger.error(f"Could not find config for platform {platform}")
sys.exit(1)
# Download the image to a temp directory
img_downloader = ImageDownloader()
img_path = img_downloader.download_image(
image, creds, selected_platform.get_config())
if not img_path:
sys.exit(1)
# Unpack the image with umoci
tag = ImageDownloader().get_image_tag(image)
img_unpacker = ImageUnpackager(src=img_path, dst=outputdir)
unpack_success = img_unpacker.unpack_image(tag, delete=True)
if not unpack_success:
sys.exit(1)
# Load app metadata
metadata_from_image = img_unpacker.get_app_metadata_from_img()
appmetadata = os.path.abspath(appmetadata) if appmetadata else None
app_metadata_dict = {}
if not metadata_from_image and not appmetadata:
# No metadata at all
logger.error(
f"Cannot find app metadata file in OCI image and none provided to BundleGen")
sys.exit(1)
if not metadata_from_image and appmetadata:
# No metadata in image, but custom file provided
if not os.path.exists(appmetadata):
logger.error(f'App metadata file {appmetadata} does not exist')
sys.exit(1)
with open(appmetadata) as metadata:
logger.debug(f"Loading metadata from {appmetadata}")
app_metadata_dict = json.load(metadata)
elif metadata_from_image and appmetadata:
# Got two options for metadata, which one do we want?
if click.confirm("Metadata found in image, but custom metadata provided. Use custom metadata?"):
with open(appmetadata) as metadata:
logger.debug(f"Loading metadata from {appmetadata}")
app_metadata_dict = json.load(metadata)
else:
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
else:
# Take metadata from image
app_metadata_dict = metadata_from_image
img_unpacker.delete_img_app_metadata()
if appid:
app_metadata_dict['id'] = appid
# Begin processing. Work in the output dir where the img was unpacked to
processor = BundleProcessor(
selected_platform.get_config(), outputdir, app_metadata_dict, nodepwalking, libmatchingmode, createmountpoints)
if not processor.check_compatibility():
# Not compatible - delete any work done so far
shutil.rmtree(outputdir)
sys.exit(2)
success = processor.begin_processing()
if not success:
logger.warning("Failed to produce bundle")
sys.exit(3)
# Processing finished, now create a tarball/ipk of the output directory
if ipk:
# create control file
Utils.create_control_file(
selected_platform.get_config(), app_metadata_dict)
Utils.create_ipk(outputdir, outputdir)
logger.success(f"Successfully generated bundle at {outputdir}.ipk")
else:
tarball_settings = processor.platform_cfg.get('tarball')
file_ownership_user = tarball_settings.get('fileOwnershipSameAsUser') if tarball_settings else None
file_mask = tarball_settings.get('fileMask') if tarball_settings else None
container_uid_gid = processor.get_real_uid_gid()
uid = container_uid_gid[0] if container_uid_gid[0] and file_ownership_user else None
gid = container_uid_gid[1] if container_uid_gid[1] and file_ownership_user else None
Utils.create_tgz(outputdir, outputdir, uid, gid, file_mask)
logger.success(f"Successfully generated bundle at {outputdir}.tar.gz")