def build_extra_vars(
name,
timezone,
language,
language_name,
wifi_pwd,
edupi,
edupi_resources,
nomad,
mathews,
wikifundi_languages,
aflatoun_languages,
kalite_languages,
packages,
admin_account,
root_partition_size,
disk_size,
):
""" extra-vars friendly format of the ansiblecube configuration """
extra_vars = {
# predefined defaults we want to superseed whichever in ansiblecube
"installer_version": get_version_str(),
"mirror": mirror,
"catalogs": CATALOGS,
"kernel_version": get_content("raspbian_image").get("kernel_version"),
"root_partition_size": root_partition_size,
"disk_size": disk_size,
"project_name": name,
"timezone": timezone,
"language": language,
"language_name": language_name,
"kalite_languages": kalite_languages,
"wikifundi_languages": wikifundi_languages,
"aflatoun_languages": aflatoun_languages,
"edupi": edupi,
"edupi_has_resources": bool(edupi_resources),
"nomad": nomad,
"mathews": mathews,
"packages": packages,
"custom_branding_path": "/tmp",
"admin_account": "admin",
"admin_password": "******",
}
if wifi_pwd:
extra_vars.update({"wpa_pass": wifi_pwd})
if admin_account is not None:
extra_vars.update(
{
"admin_account": admin_account["login"],
"admin_password": admin_account["pwd"],
}
)
secret_keys = ["admin_password"]
else:
secret_keys = []
return extra_vars, secret_keys
def reset_cache(logger, build_folder, cache_folder, **kwargs):
""" wipe out the cache folder, optionnaly keeping latest master """
logger.step("Reseting cache folder: {}".format(cache_folder))
cache_size, free_space = display_cache_and_free_space(
logger, build_folder, cache_folder
)
logger.std("-------------")
if kwargs.get("keep_master"):
tmp_master_fpath = None
master = get_content("hotspot_master_image")
master_fpath = os.path.join(cache_folder, master["name"])
if (
os.path.exists(master_fpath)
and get_checksum(master_fpath) == master["checksum"]
):
# latest master to be moved temporarly to build-dir
tmp_master_fpath = os.path.join(
build_folder, ".__tmp--{}".format(master["name"])
)
logger.std("Keeping your latest master aside: {}".format(master["name"]))
try:
shutil.move(master_fpath, tmp_master_fpath)
except Exception as exp:
logger.err("Unable to move your latest master into build-dir. Exiting.")
return 1
logger.std("Removing cache...", end="")
try:
shutil.rmtree(cache_folder)
except Exception as exp:
logger.err("FAILED ({}).".format(exp))
else:
logger.succ("OK.")
logger.std("Recreating cache placeholder.")
cache_folder = get_cache(build_folder)
if kwargs.get("keep_master"):
logger.std("Restoring your latest master.")
try:
shutil.move(tmp_master_fpath, master_fpath)
except Exception as exp:
logger.err("Unable to move back your master file into fresh cache.")
if tmp_master_fpath is not None:
logger.err("Please find your master at: {}".format(tmp_master_fpath))
return 1
logger.std("-------------")
display_cache_and_free_space(
logger, build_folder, cache_folder, cache_size, free_space
)
return 0
def run_installation(
name,
timezone,
language,
wifi_pwd,
admin_account,
kalite,
aflatoun,
wikifundi,
edupi,
edupi_resources,
nomad,
mathews,
africatik,
zim_install,
size,
logger,
cancel_event,
sd_card,
favicon,
logo,
css,
done_callback=None,
build_dir=".",
filename=None,
qemu_ram="2G",
shrink=False,
):
logger.start(bool(sd_card))
logger.stage("init")
cache_folder = get_cache(build_dir)
try:
logger.std("Preventing system from sleeping")
sleep_ref = prevent_sleep(logger)
logger.step("Check System Requirements")
logger.std("Please read {} for details".format(requirements_url))
sysreq_ok, missing_deps = host_matches_requirements(build_dir)
if not sysreq_ok:
raise SystemError(
"Your system does not matches system requirements:\n{}".format(
"\n".join([" - {}".format(dep) for dep in missing_deps])))
logger.step("Ensure user files are present")
for user_fpath in (edupi_resources, favicon, logo, css):
if (user_fpath is not None and not isremote(user_fpath)
and not os.path.exists(user_fpath)):
raise ValueError(
"Specified file is not available ({})".format(user_fpath))
logger.step("Prepare Image file")
# set image names
if not filename:
filename = "hotspot-{}".format(
datetime.today().strftime("%Y_%m_%d-%H_%M_%S"))
image_final_path = os.path.join(build_dir, filename + ".img")
image_building_path = os.path.join(build_dir,
filename + ".BUILDING.img")
image_error_path = os.path.join(build_dir, filename + ".ERROR.img")
# loop device mode on linux (for mkfs in userspace)
if sys.platform == "linux":
loop_dev = guess_next_loop_device(logger)
if loop_dev and not can_write_on(loop_dev):
logger.step("Change loop device mode ({})".format(sd_card))
previous_loop_mode = allow_write_on(loop_dev, logger)
else:
previous_loop_mode = None
base_image = get_content("hotspot_master_image")
# harmonize options
packages = [] if zim_install is None else zim_install
kalite_languages = [] if kalite is None else kalite
wikifundi_languages = [] if wikifundi is None else wikifundi
aflatoun_languages = ["fr", "en"] if aflatoun else []
if edupi_resources and not isremote(edupi_resources):
logger.step("Copying EduPi resources into cache")
shutil.copy(edupi_resources, cache_folder)
# prepare ansible options
ansible_options = {
"name": name,
"timezone": timezone,
"language": language,
"language_name": dict(data.hotspot_languages)[language],
"edupi": edupi,
"edupi_resources": edupi_resources,
"nomad": nomad,
"mathews": mathews,
"africatik": africatik,
"wikifundi_languages": wikifundi_languages,
"aflatoun_languages": aflatoun_languages,
"kalite_languages": kalite_languages,
"packages": packages,
"wifi_pwd": wifi_pwd,
"admin_account": admin_account,
"disk_size": size,
"root_partition_size": base_image.get("root_partition_size"),
}
extra_vars, secret_keys = ansiblecube.build_extra_vars(
**ansible_options)
# display config in log
logger.step("Dumping Hotspot Configuration")
logger.raw_std(
json.dumps(
{
k: "****" if k in secret_keys else v
for k, v in extra_vars.items()
},
indent=4,
))
# gen homepage HTML
homepage_path = save_homepage(
generate_homepage(logger, ansible_options))
logger.std("homepage saved to: {}".format(homepage_path))
# Download Base image
logger.stage("master")
logger.step("Retrieving base image file")
rf = download_content(base_image, logger, build_dir)
if not rf.successful:
logger.err(
"Failed to download base image.\n{e}".format(e=rf.exception))
sys.exit(1)
elif rf.found:
logger.std("Reusing already downloaded base image ZIP file")
logger.progress(0.5)
# extract base image and rename
logger.step("Extracting base image from ZIP file")
unzip_file(
archive_fpath=rf.fpath,
src_fname=base_image["name"].replace(".zip", ""),
build_folder=build_dir,
dest_fpath=image_building_path,
)
logger.std("Extraction complete: {p}".format(p=image_building_path))
logger.progress(0.9)
if not os.path.exists(image_building_path):
raise IOError(
"image path does not exists: {}".format(image_building_path))
logger.step("Testing mount procedure")
if not test_mount_procedure(image_building_path, logger, True):
raise ValueError("thorough mount procedure failed")
# collection contains both downloads and processing callbacks
# for all requested contents
collection = get_collection(
edupi=edupi,
edupi_resources=edupi_resources,
nomad=nomad,
mathews=mathews,
africatik=africatik,
packages=packages,
kalite_languages=kalite_languages,
wikifundi_languages=wikifundi_languages,
aflatoun_languages=aflatoun_languages,
)
# download contents into cache
logger.stage("download")
logger.step("Starting all content downloads")
downloads = list(get_all_contents_for(collection))
archives_total_size = sum([c["archive_size"] for c in downloads])
retrieved = 0
for dl_content in downloads:
logger.step("Retrieving {name} ({size})".format(
name=dl_content["name"],
size=human_readable_size(dl_content["archive_size"]),
))
rf = download_content(dl_content, logger, build_dir)
if not rf.successful:
logger.err("Error downloading {u} to {p}\n{e}".format(
u=dl_content["url"], p=rf.fpath, e=rf.exception))
raise rf.exception if rf.exception else IOError
elif rf.found:
logger.std("Reusing already downloaded {p}".format(p=rf.fpath))
else:
logger.std("Saved `{p}` successfuly: {s}".format(
p=dl_content["name"],
s=human_readable_size(rf.downloaded_size)))
retrieved += dl_content["archive_size"]
logger.progress(retrieved, archives_total_size)
# check edupi resources compliance
if edupi_resources:
logger.step("Verifying EduPi resources file names")
exfat_compat, exfat_errors = ensure_zip_exfat_compatible(
get_content_cache(get_alien_content(edupi_resources),
cache_folder, True))
if not exfat_compat:
raise ValueError("Your EduPi resources archive is incorrect.\n"
"It should be a ZIP file of a root folder "
"in which all files have exfat-compatible "
"names (no {chars})\n... {fnames}".format(
chars=" ".join(EXFAT_FORBIDDEN_CHARS),
fnames="\n... ".join(exfat_errors),
))
else:
logger.std("EduPi resources archive OK")
# instanciate emulator
logger.stage("setup")
logger.step("Preparing qemu VM")
emulator = qemu.Emulator(
data.vexpress_boot_kernel,
data.vexpress_boot_dtb,
image_building_path,
logger,
ram=qemu_ram,
)
# Resize image
logger.step("Resizing image file from {s1} to {s2}".format(
s1=human_readable_size(emulator.get_image_size()),
s2=human_readable_size(size),
))
if size < emulator.get_image_size():
logger.err("cannot decrease image size")
raise ValueError("cannot decrease image size")
emulator.resize_image(size)
# Run emulation
logger.step("Starting-up VM (first-time)")
with emulator.run(cancel_event) as emulation:
# copying ansiblecube again into the VM
# should the master-version been updated
logger.step("Copy ansiblecube")
emulation.exec_cmd("sudo /bin/rm -rf {}".format(
ansiblecube.ansiblecube_path))
emulation.put_dir(data.ansiblecube_path,
ansiblecube.ansiblecube_path)
logger.step("Run ansiblecube for `resize`")
ansiblecube.run(emulation, ["resize"], extra_vars, secret_keys)
logger.step("Starting-up VM (second-time)")
with emulator.run(cancel_event) as emulation:
logger.step("Run ansiblecube phase I")
ansiblecube.run_phase_one(
emulation,
extra_vars,
secret_keys,
homepage=homepage_path,
logo=logo,
favicon=favicon,
css=css,
)
# wait for QEMU to release file (windows mostly)
time.sleep(10)
# mount image's 3rd partition on host
logger.stage("copy")
logger.step("Formating data partition on host")
format_data_partition(image_building_path, logger)
logger.step("Mounting data partition on host")
# copy contents from cache to mount point
try:
mount_point, device = mount_data_partition(image_building_path,
logger)
logger.step("Processing downloaded content onto data partition")
expanded_total_size = sum([c["expanded_size"] for c in downloads])
processed = 0
for category, content_dl_cb, content_run_cb, cb_kwargs in collection:
logger.step("Processing {cat}".format(cat=category))
content_run_cb(cache_folder=cache_folder,
mount_point=mount_point,
logger=logger,
**cb_kwargs)
# size of expanded files for this category (for progress)
processed += sum(
[c["expanded_size"] for c in content_dl_cb(**cb_kwargs)])
logger.progress(processed, expanded_total_size)
except Exception as exp:
try:
unmount_data_partition(mount_point, device, logger)
except NameError:
pass # if mount_point or device are not defined
raise exp
time.sleep(10)
# unmount partition
logger.step("Unmounting data partition")
unmount_data_partition(mount_point, device, logger)
time.sleep(10)
# rerun emulation for discovery
logger.stage("move")
logger.step("Starting-up VM (third-time)")
with emulator.run(cancel_event) as emulation:
logger.step("Run ansiblecube phase II")
ansiblecube.run_phase_two(emulation, extra_vars, secret_keys)
if shrink:
logger.step("Shrink size of physical image file")
# calculate physical size of image
required_image_size = get_required_image_size(collection)
if required_image_size + ONE_GB >= size:
# less than 1GB difference, don't bother
pass
else:
# set physical size to required + margin
physical_size = math.ceil(
required_image_size / ONE_GB) * ONE_GB
emulator.resize_image(physical_size, shrink=True)
# wait for QEMU to release file (windows mostly)
logger.succ("Image creation successful.")
time.sleep(20)
except Exception as e:
logger.failed(str(e))
# display traceback on logger
logger.std("\n--- Exception Trace ---\n{exp}\n---".format(
exp=traceback.format_exc()))
# Set final image filename
if os.path.isfile(image_building_path):
os.rename(image_building_path, image_error_path)
error = e
else:
try:
# Set final image filename
tries = 0
while True:
try:
os.rename(image_building_path, image_final_path)
except Exception as exp:
logger.err(exp)
tries += 1
if tries > 3:
raise exp
time.sleep(5 * tries)
continue
else:
logger.std(
"Renamed image file to {}".format(image_final_path))
break
# Write image to SD Card
if sd_card:
logger.stage("write")
logger.step("Writting image to SD-card ({})".format(sd_card))
try:
etcher_writer = EtcherWriterThread(args=(image_final_path,
sd_card, logger))
cancel_event.register_thread(thread=etcher_writer)
etcher_writer.start()
etcher_writer.join(timeout=2) # make sure it started
while etcher_writer.is_alive():
pass
logger.std("not alive")
etcher_writer.join(timeout=2)
cancel_event.unregister_thread()
if etcher_writer.exp is not None:
raise etcher_writer.exp
logger.std("Done writing and verifying.")
time.sleep(5)
except Exception:
logger.succ("Image created successfuly.")
logger.err(
"Writing or verification of Image to your SD-card failed.\n"
"Please use a third party tool to flash your image "
"onto your SD-card. See File menu for links to Etcher."
)
raise Exception("Failed to write Image to SD-card")
except Exception as e:
logger.failed(str(e))
# display traceback on logger
logger.std("\n--- Exception Trace ---\n{exp}\n---".format(
exp=traceback.format_exc()))
error = e
else:
logger.complete()
error = None
finally:
logger.std("Restoring system sleep policy")
restore_sleep_policy(sleep_ref, logger)
if sys.platform == "linux" and loop_dev and previous_loop_mode:
logger.step("Restoring loop device ({}) mode".format(loop_dev))
restore_mode(loop_dev, previous_loop_mode, logger)
# display durations summary
logger.summary()
if done_callback:
done_callback(error)
return error
def get_virtual_device(image_fpath, logger):
""" create and return a loop device or drive letter we can format/mount """
if sys.platform == "linux":
# find out offset for third partition from the root part size
base_image = get_content("hotspot_master_image")
disk_size = get_qemu_image_size(image_fpath, logger)
offset, size = get_start_offset(base_image.get("root_partition_size"),
disk_size)
# prepare loop device
if bool(os.getenv("NO_UDISKS", False)):
loop_maker = subprocess_pretty_call(
[
"/sbin/losetup",
"--offset",
str(offset),
"--sizelimit",
str(size),
"--find",
"--show",
image_fpath,
],
logger,
check=True,
decode=True,
)[0].strip()
else:
loop_maker = subprocess_pretty_call(
[
udisksctl_exe,
"loop-setup",
"--offset",
str(offset),
"--size",
str(size),
"--file",
image_fpath,
udisks_nou,
],
logger,
check=True,
decode=True,
)[0].strip()
target_dev = re.search(r"(\/dev\/loop[0-9]+)\.?$",
loop_maker).groups()[0]
elif sys.platform == "darwin":
# attach image to create loop devices
hdiutil_out = subprocess_pretty_call(
[hdiutil_exe, "attach", "-nomount", image_fpath],
logger,
check=True,
decode=True,
)[0].strip()
target_dev = str(hdiutil_out.splitlines()[0].split()[0])
elif sys.platform == "win32":
# make sure we have imdisk installed
install_imdisk(logger)
# get an available letter
target_dev = get_avail_drive_letter(logger)
return target_dev
def main(logger,
disk_size,
root_size,
build_folder,
qemu_ram,
image_fname=None):
# convert sizes to bytes and make sure those are usable
try:
root_size = int(root_size) * ONE_GB
disk_size = get_adjusted_image_size(int(disk_size) * ONE_GB)
if root_size < MIN_ROOT_SIZE:
raise ValueError("root partition must be at least {}".format(
human_readable_size(MIN_ROOT_SIZE, False)))
if root_size >= disk_size:
raise ValueError("root partition must be smaller than disk size")
except Exception as exp:
logger.err("Erroneous size option: {}".format(exp))
sys.exit(1)
logger.step("Starting master creation: {} ({} root)".format(
human_readable_size(disk_size, False),
human_readable_size(root_size, False)))
# default output file name
if image_fname is None:
image_fname = "hotspot-master_{date}.img".format(
date=datetime.datetime.now().strftime("%Y-%m-%d"))
image_fpath = os.path.join(build_folder, image_fname)
logger.step("starting with target: {}".format(image_fpath))
# download raspbian
logger.step("Retrieving raspbian image file")
raspbian_image = get_content("raspbian_image")
rf = download_content(raspbian_image, logger, build_folder)
if not rf.successful:
logger.err("Failed to download raspbian.\n{e}".format(e=rf.exception))
sys.exit(1)
elif rf.found:
logger.std("Reusing already downloaded raspbian ZIP file")
# extract raspbian and rename
logger.step("Extracting raspbian image from ZIP file")
unzip_file(
archive_fpath=rf.fpath,
src_fname=raspbian_image["name"].replace(".zip", ".img"),
build_folder=build_folder,
dest_fpath=image_fpath,
)
logger.std("Extraction complete: {p}".format(p=image_fpath))
if not os.path.exists(image_fpath):
raise IOError("image path does not exists: {}".format(image_fpath))
error = run_in_qemu(image_fpath, disk_size, root_size, logger,
CancelEvent(), qemu_ram)
if error:
logger.err("ERROR: unable to properly create image: {}".format(error))
sys.exit(1)
logger.std("SUCCESS! {} was built successfuly".format(image_fpath))
wikifundi_languages=args.wikifundi,
aflatoun_languages=["fr", "en"] if args.aflatoun == "yes" else [],
)
cache_folder = get_cache(args.build_dir)
# how much space is available on the build directory?
avail_space_in_build_dir = get_free_space_in_dir(args.build_dir)
try:
# how much space do we need to build the image?
space_required_to_build = get_required_building_space(
collection, cache_folder, args.output_size)
# how large should the image be?
required_image_size = get_required_image_size(collection)
except FileNotFoundError as exp:
print("Supplied File Not Found: {}".format(exp.filename), file=sys.stderr)
sys.exit(1)
base_image_size = get_content("hotspot_master_image")["expanded_size"]
if args.size < base_image_size:
print(
"image size can not be under {size}".format(
size=human_readable_size(base_image_size, False)),
file=sys.stderr,
)
sys.exit(3)
if args.output_size < required_image_size:
print(
"image size ({img}/{img2}) is not large enough for the content ({req})"
.format(
img=human_readable_size(args.size, False),
img2=human_readable_size(args.output_size, False),