def test_human_readable_size():
assert human_readable_size(512) == "512.0B"
assert human_readable_size(512 * KiB) == "512.0KiB"
assert human_readable_size(512 * MiB) == "512.0MiB"
assert human_readable_size(512 * GiB) == "512.0GiB"
assert human_readable_size(512 * TiB) == "512.0TiB"
assert human_readable_size(512 * PiB) == "512.0PiB"
assert human_readable_size(512 * EiB) == "512.0EiB"
assert human_readable_size(512 * ZiB) == "512.0ZiB"
assert human_readable_size(512 * YiB) == "512.0YiB"
assert human_readable_size(51200 * YiB) == "51200.0YiB"
def update_folder(self):
'''更新文件浏览窗口'''
ls_file = "ls.txt"
reply = self.client.ls(local_file=ls_file) #获取当前文件夹下的所有文件信息
if reply == True:
self.folderWidget.clearContents()
self.folderWidget.setRowCount(len(open(ls_file,'r').readlines()))
i = 0
with open(ls_file, "r") as f:
for line in f:
list = line.split()
#文件名
name_item = QTableWidgetItem(list[8])
name_item.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled)
self.folderWidget.setItem(i, 0, name_item)
#文件类型
type = get_file_type(file_name=list[8], access_string=list[0])
type_item = QTableWidgetItem(type)
type_item.setFlags(Qt.NoItemFlags)
self.folderWidget.setItem(i, 1, type_item)
#文件大小
if type == "folder":
size = "--"
else:
size = human_readable_size(list[4])
size_item = QTableWidgetItem(size)
size_item.setFlags(Qt.NoItemFlags)
self.folderWidget.setItem(i, 2, size_item)
#文件的最近修改时间
time_item = QTableWidgetItem(" ".join(list[5:8]))
time_item.setFlags(Qt.NoItemFlags)
self.folderWidget.setItem(i, 3, time_item)
i += 1
def _set_ram(self, requested_ram):
""" applies requested RAM to qemu if it's available otherwise less """
# less than a GB is very short
if host_ram / ONE_GiB <= 1.0:
self._ram = "256m"
return
# at most, use RAM minus 512m
max_ram = int(host_ram - ONE_GiB / 2)
# parse specified ram in Mega or Giga
if re.match(r"\d+[mg]$", requested_ram):
ram_amount, ram_unit = int(requested_ram[:-1]), requested_ram[-1]
else:
# no unit, assuming M
ram_amount, ram_unit = int(requested_ram), "m"
if ram_unit == "g":
ram_amount = ram_amount * (ONE_GiB)
# use requested if it doesn't exceed max_ram
ram = max_ram if ram_amount > max_ram else ram_amount
# vexpress-a15 is capped at 30G
if int(ram / ONE_GiB) > 30:
ram = 30 * ONE_GiB
self._ram = "{ram}M".format(ram=int(ram / ONE_MiB))
self._logger.std(
" using {ram} RAM".format(ram=human_readable_size(ram)))
def get_iterator():
devices = []
plist = plistlib.loads(subprocess.check_output(["diskutil", "list", "-plist"]))
device_names = []
for name in plist["AllDisks"]:
device_names.extend(re.findall(r"^(disk\d+)$", name))
for name in device_names:
plist = plistlib.loads(subprocess.check_output(["diskutil", "info", "-plist", name]))
size = plist["Size"]
formatted_size = human_readable_size(size)
devices.append({
"bus_protocol": plist["BusProtocol"],
"device_identifier": plist["DeviceIdentifier"],
"device": plist["DeviceNode"],
"io_registry_entry_name": plist["IORegistryEntryName"],
"media_name": plist["MediaName"],
"media_type": plist["MediaType"],
"removable": plist["Removable"],
"size": size,
"formatted_size": formatted_size,
"volume_name": plist["VolumeName"],
})
return filter(lambda d: d["removable"] and d["size"] != 0 and d["bus_protocol"] != "Disk Image", devices)
def get_iterator():
devices = []
bus = dbus.SystemBus()
udisk = bus.get_object('org.freedesktop.UDisks2', '/org/freedesktop/UDisks2')
udisk_interface = dbus.Interface(udisk, 'org.freedesktop.DBus.ObjectManager')
for key, value in udisk_interface.GetManagedObjects().items():
info = value.get('org.freedesktop.UDisks2.Block', {})
if info.get('IdUsage') == "" and info.get('Drive') != "/":
device = bytes(info.get('PreferredDevice')).decode('utf-8')
size = info.get('Size')
formatted_size = human_readable_size(size)
devices.append({
"key": key,
# Because device name ends with \x00
"device": device.replace("\x00", ""),
"size": size,
"formatted_size": formatted_size,
"id_label": info.get('IdLabel'),
"drive_key": info.get('Drive'),
"id": info.get('Id'),
})
for drive_key, drive_value in udisk_interface.GetManagedObjects().items():
info = drive_value.get('org.freedesktop.UDisks2.Drive', {})
for block in devices:
if drive_key == block["drive_key"]:
block["drive_removable"] = info.get('Removable')
block["drive_id"] = info.get('Id')
block["drive_connection_bus"] = info.get('ConnectionBus')
return filter(lambda d: d["drive_removable"] and d["size"] != 0, devices)
def get_iterator():
lines = subprocess.check_output(["wmic", "diskdrive"], **startup_info_args()).decode('utf-8').splitlines()
column = {}
matches = re.finditer(r"(\w+\W+)", lines[0])
for match in matches:
column[lines[0][match.start():match.end()].strip()] = match
devices = []
lines.pop(0)
for line in filter(lambda l: len(l) is not 0, lines):
size = extract_field(column["Size"], line)
formatted_size = human_readable_size(size)
devices.append({
"caption": extract_field(column["Caption"], line),
"description": extract_field(column["Description"], line),
"device": extract_field(column["DeviceID"], line),
"media_type": extract_field(column["MediaType"], line),
"model": extract_field(column["Model"], line),
"name": extract_field(column["Name"], line),
"size": size,
"formatted_size": formatted_size,
})
return filter(lambda d: d["media_type"] != "Fixed hard disk media" and d["size"] is not '', devices)
def list_cache_files(logger, build_folder, cache_folder, **kwargs):
""" colored list of all files in cache with legend (to Keep or to Remove) """
logger.step("Listing cache content for: {}".format(cache_folder))
cfiles = get_analyzed_cache_files(logger, cache_folder)
cache_size, nb_files, free_space = get_cache_size_and_free_space(
build_folder, cache_folder
)
logger.std("-------------\nLEGEND\n-------------")
logger.std("K: to be kept (latest content or alien folder)")
logger.std("R: to be removed (obsolete, damaged or alien file)")
logger.std("F: is a file")
logger.std("D: is a directory")
logger.std("-------------")
claimable = 0
for cfile in cfiles:
if cfile["latest"]:
fmt = logger.succ
elif not cfile["alien"]:
fmt = logger.err
claimable += cfile["size"]
else:
fmt = logger.std
fmt(
" {a} [{s}]\t\t{t} {f}{sf}".format(
a="K" if cfile["keep"] else "R",
s=human_readable_size(cfile["size"]),
f=cfile["fname"],
t="D" if cfile["isdir"] else "F",
sf=" --- [{}]".format(cfile["latest"]) if cfile["latest"] else "",
)
)
logger.std("-------------")
logger.std("TOTAL USED SPACE in cache: {}".format(human_readable_size(cache_size)))
logger.std("TOTAL FREE SPACE in cache: {}".format(human_readable_size(free_space)))
logger.std(
"TOTAL CLAIMABLE SPACE in cache: {}".format(human_readable_size(claimable))
)
return 0
def scan_files(self, paths):
if self.use_tqdm:
paths = tqdm(paths, unit="files")
for file_path in paths:
with open(file_path, "rb") as f:
data = f.read()
self.scan_buf(data)
if self.use_tqdm:
paths.set_postfix(
{'waste': util.human_readable_size(self.total_waste)})
def display_cache_and_free_space(
logger, build_folder, cache_folder, old_cache_size=None, old_free_space=None
):
""" display cache size and free space shortcut """
cache_size, nb_files, free_space = get_cache_size_and_free_space(
build_folder, cache_folder
)
if old_cache_size:
logger.std(
"NEW CACHE SIZE: {} (removed {})".format(
human_readable_size(cache_size),
human_readable_size(old_cache_size - cache_size),
)
)
else:
logger.std(
"CACHE SIZE: {} ({} files)".format(
human_readable_size(cache_size), nb_files
)
)
if old_free_space:
logger.std(
"NEW FREE SPACE: {} (reclaimed {})".format(
human_readable_size(free_space),
human_readable_size(free_space - old_free_space),
)
)
else:
logger.std("FREE SPACE: {}".format(human_readable_size(free_space)))
return cache_size, free_space
def update_free_space(self):
free_space = self.get_free_space()
human_readable_free_space = human_readable_size(free_space)
self.component.free_space_label1.set_text(human_readable_free_space)
self.component.free_space_label2.set_text(human_readable_free_space)
condition = free_space >= 0
validate_label(self.component.free_space_label1, condition)
validate_label(self.component.free_space_label2, condition)
for row in self.component.zim_list_store:
if free_space - int(row[9]) >= 0:
row[11] = VALID_RGBA
else:
row[11] = INVALID_RGBA
return free_space
def run_in_qemu(image_fpath, disk_size, root_size, logger, cancel_event,
qemu_ram):
logger.step("starting QEMU")
try:
# Instance emulator
emulator = qemu.Emulator(
data.vexpress_boot_kernel,
data.vexpress_boot_dtb,
image_fpath,
logger,
ram=qemu_ram,
is_master=True,
)
logger.step("resizing QEMU image to {}GB".format(
human_readable_size(disk_size, False)))
emulator.resize_image(disk_size)
# Run emulation
with emulator.run(cancel_event) as emulation:
# enable SSH for good
logger.step("Enable SSH")
emulation.exec_cmd("sudo /bin/systemctl enable ssh")
logger.step("Copy ansiblecube")
ansible_root_path = os.path.dirname(ansiblecube_emulation_path)
emulation.exec_cmd(
"sudo mkdir --mode 0755 -p {}".format(ansible_root_path))
emulation.put_dir(data.ansiblecube_path,
ansiblecube_emulation_path)
emulation.exec_cmd(
"sudo chown pi:pi -R {}".format(ansible_root_path))
# Run ansiblecube
logger.step("Run ansiblecube")
run_for_image(machine=emulation,
root_partition_size=root_size,
disk_size=disk_size)
except Exception as e:
return e
return None
def get_iterator():
devices = []
bus = dbus.SystemBus()
udisk = bus.get_object("org.freedesktop.UDisks2",
"/org/freedesktop/UDisks2")
udisk_interface = dbus.Interface(udisk,
"org.freedesktop.DBus.ObjectManager")
for key, value in udisk_interface.GetManagedObjects().items():
info = value.get("org.freedesktop.UDisks2.Block", {})
if info.get("IdUsage") == "" and info.get("Drive") != "/":
device = bytes(info.get("PreferredDevice")).decode("utf-8")
size = info.get("Size")
formatted_size = human_readable_size(size, binary=False)
devices.append({
"key": key,
# Because device name ends with \x00
"device": device.replace("\x00", ""),
"size": size,
"formatted_size": formatted_size,
"id_label": info.get("IdLabel"),
"drive_key": info.get("Drive"),
"id": info.get("Id"),
})
for drive_key, drive_value in udisk_interface.GetManagedObjects(
).items():
info = drive_value.get("org.freedesktop.UDisks2.Drive", {})
for block in devices:
if drive_key == block["drive_key"]:
block["drive_removable"] = info.get("Removable")
block["drive_id"] = info.get("Id")
block["drive_connection_bus"] = info.get("ConnectionBus")
return filter(lambda d: d["drive_removable"] and d["size"] != 0,
devices)
def test():
logger = logging.LoggerAdapter(logging.getLogger(), {"caller": __name__})
src_image_folder = "test/images/*.png"
src_images = []
for src_img in glob.glob(src_image_folder):
src_images.append(src_img)
payloads_folder = "test/payloads/*.test"
payloads = []
for payload in glob.glob(payloads_folder):
payloads.append(payload)
tmp_payload_output_path = "test/test_payload_extracted"
tmp_output_image = "test/test_processed_image.png"
test_methods = ["LSB", "PVD"]
logger.info("Beginning StegArmory test!")
total_iterations = len(src_images) * len(payloads) * len(test_methods)
logger.info("Running %d test cases!" % total_iterations)
results = []
i = 0
for src_img in src_images:
for payload in payloads:
for method in test_methods:
src_img_filename = os.path.basename(src_img)
payload_filename = os.path.basename(payload)
tmp_output_img_filename = os.path.basename(tmp_output_image)
logger.info("Testing %s embed on image %s with payload %s" %
(method, src_img_filename, payload_filename))
if method == "LSB":
proc = processor.LSBProcessor(src_img)
elif method == "PVD":
proc = processor.PVDProcessor(src_img)
embed_time_a = time.perf_counter()
proc.embed_payload(payload, tmp_output_image)
embed_time_b = time.perf_counter()
embed_time = round(embed_time_b - embed_time_a, 4)
logger.info(
"Testing %s extract on image %s with payload %s" %
(method, tmp_output_img_filename, payload_filename))
proc2 = type(proc)(tmp_output_image)
extract_time_a = time.perf_counter()
proc2.extract_payload(tmp_payload_output_path)
extract_time_b = time.perf_counter()
extract_time = round(extract_time_b - extract_time_a, 4)
proc.compare(proc2)
results.append([
src_img, method, payload, embed_time, extract_time,
util.human_readable_size(proc.max_payload_size // 8,
2), proc.comparison_stats['psnr'],
proc.comparison_stats['ssim']
])
i += 1
logger.info("Overall Progress: %d/%d (%s%%)" %
(i, total_iterations,
str(round((i / total_iterations) * 100, 2))))
print(
tabulate.tabulate(results,
headers=[
"Source Image", "Method", "Payload",
"Embed Time", "Extract Time", "Max Payload Size",
"PSNR", "SSIM"
]))
def run_installation(
name,
timezone,
language,
wifi_pwd,
admin_account,
kalite,
aflatoun,
wikifundi,
edupi,
edupi_resources,
nomad,
mathews,
africatik,
zim_install,
size,
logger,
cancel_event,
sd_card,
favicon,
logo,
css,
done_callback=None,
build_dir=".",
filename=None,
qemu_ram="2G",
shrink=False,
):
logger.start(bool(sd_card))
logger.stage("init")
cache_folder = get_cache(build_dir)
try:
logger.std("Preventing system from sleeping")
sleep_ref = prevent_sleep(logger)
logger.step("Check System Requirements")
logger.std("Please read {} for details".format(requirements_url))
sysreq_ok, missing_deps = host_matches_requirements(build_dir)
if not sysreq_ok:
raise SystemError(
"Your system does not matches system requirements:\n{}".format(
"\n".join([" - {}".format(dep) for dep in missing_deps])))
logger.step("Ensure user files are present")
for user_fpath in (edupi_resources, favicon, logo, css):
if (user_fpath is not None and not isremote(user_fpath)
and not os.path.exists(user_fpath)):
raise ValueError(
"Specified file is not available ({})".format(user_fpath))
logger.step("Prepare Image file")
# set image names
if not filename:
filename = "hotspot-{}".format(
datetime.today().strftime("%Y_%m_%d-%H_%M_%S"))
image_final_path = os.path.join(build_dir, filename + ".img")
image_building_path = os.path.join(build_dir,
filename + ".BUILDING.img")
image_error_path = os.path.join(build_dir, filename + ".ERROR.img")
# loop device mode on linux (for mkfs in userspace)
if sys.platform == "linux":
loop_dev = guess_next_loop_device(logger)
if loop_dev and not can_write_on(loop_dev):
logger.step("Change loop device mode ({})".format(sd_card))
previous_loop_mode = allow_write_on(loop_dev, logger)
else:
previous_loop_mode = None
base_image = get_content("hotspot_master_image")
# harmonize options
packages = [] if zim_install is None else zim_install
kalite_languages = [] if kalite is None else kalite
wikifundi_languages = [] if wikifundi is None else wikifundi
aflatoun_languages = ["fr", "en"] if aflatoun else []
if edupi_resources and not isremote(edupi_resources):
logger.step("Copying EduPi resources into cache")
shutil.copy(edupi_resources, cache_folder)
# prepare ansible options
ansible_options = {
"name": name,
"timezone": timezone,
"language": language,
"language_name": dict(data.hotspot_languages)[language],
"edupi": edupi,
"edupi_resources": edupi_resources,
"nomad": nomad,
"mathews": mathews,
"africatik": africatik,
"wikifundi_languages": wikifundi_languages,
"aflatoun_languages": aflatoun_languages,
"kalite_languages": kalite_languages,
"packages": packages,
"wifi_pwd": wifi_pwd,
"admin_account": admin_account,
"disk_size": size,
"root_partition_size": base_image.get("root_partition_size"),
}
extra_vars, secret_keys = ansiblecube.build_extra_vars(
**ansible_options)
# display config in log
logger.step("Dumping Hotspot Configuration")
logger.raw_std(
json.dumps(
{
k: "****" if k in secret_keys else v
for k, v in extra_vars.items()
},
indent=4,
))
# gen homepage HTML
homepage_path = save_homepage(
generate_homepage(logger, ansible_options))
logger.std("homepage saved to: {}".format(homepage_path))
# Download Base image
logger.stage("master")
logger.step("Retrieving base image file")
rf = download_content(base_image, logger, build_dir)
if not rf.successful:
logger.err(
"Failed to download base image.\n{e}".format(e=rf.exception))
sys.exit(1)
elif rf.found:
logger.std("Reusing already downloaded base image ZIP file")
logger.progress(0.5)
# extract base image and rename
logger.step("Extracting base image from ZIP file")
unzip_file(
archive_fpath=rf.fpath,
src_fname=base_image["name"].replace(".zip", ""),
build_folder=build_dir,
dest_fpath=image_building_path,
)
logger.std("Extraction complete: {p}".format(p=image_building_path))
logger.progress(0.9)
if not os.path.exists(image_building_path):
raise IOError(
"image path does not exists: {}".format(image_building_path))
logger.step("Testing mount procedure")
if not test_mount_procedure(image_building_path, logger, True):
raise ValueError("thorough mount procedure failed")
# collection contains both downloads and processing callbacks
# for all requested contents
collection = get_collection(
edupi=edupi,
edupi_resources=edupi_resources,
nomad=nomad,
mathews=mathews,
africatik=africatik,
packages=packages,
kalite_languages=kalite_languages,
wikifundi_languages=wikifundi_languages,
aflatoun_languages=aflatoun_languages,
)
# download contents into cache
logger.stage("download")
logger.step("Starting all content downloads")
downloads = list(get_all_contents_for(collection))
archives_total_size = sum([c["archive_size"] for c in downloads])
retrieved = 0
for dl_content in downloads:
logger.step("Retrieving {name} ({size})".format(
name=dl_content["name"],
size=human_readable_size(dl_content["archive_size"]),
))
rf = download_content(dl_content, logger, build_dir)
if not rf.successful:
logger.err("Error downloading {u} to {p}\n{e}".format(
u=dl_content["url"], p=rf.fpath, e=rf.exception))
raise rf.exception if rf.exception else IOError
elif rf.found:
logger.std("Reusing already downloaded {p}".format(p=rf.fpath))
else:
logger.std("Saved `{p}` successfuly: {s}".format(
p=dl_content["name"],
s=human_readable_size(rf.downloaded_size)))
retrieved += dl_content["archive_size"]
logger.progress(retrieved, archives_total_size)
# check edupi resources compliance
if edupi_resources:
logger.step("Verifying EduPi resources file names")
exfat_compat, exfat_errors = ensure_zip_exfat_compatible(
get_content_cache(get_alien_content(edupi_resources),
cache_folder, True))
if not exfat_compat:
raise ValueError("Your EduPi resources archive is incorrect.\n"
"It should be a ZIP file of a root folder "
"in which all files have exfat-compatible "
"names (no {chars})\n... {fnames}".format(
chars=" ".join(EXFAT_FORBIDDEN_CHARS),
fnames="\n... ".join(exfat_errors),
))
else:
logger.std("EduPi resources archive OK")
# instanciate emulator
logger.stage("setup")
logger.step("Preparing qemu VM")
emulator = qemu.Emulator(
data.vexpress_boot_kernel,
data.vexpress_boot_dtb,
image_building_path,
logger,
ram=qemu_ram,
)
# Resize image
logger.step("Resizing image file from {s1} to {s2}".format(
s1=human_readable_size(emulator.get_image_size()),
s2=human_readable_size(size),
))
if size < emulator.get_image_size():
logger.err("cannot decrease image size")
raise ValueError("cannot decrease image size")
emulator.resize_image(size)
# Run emulation
logger.step("Starting-up VM (first-time)")
with emulator.run(cancel_event) as emulation:
# copying ansiblecube again into the VM
# should the master-version been updated
logger.step("Copy ansiblecube")
emulation.exec_cmd("sudo /bin/rm -rf {}".format(
ansiblecube.ansiblecube_path))
emulation.put_dir(data.ansiblecube_path,
ansiblecube.ansiblecube_path)
logger.step("Run ansiblecube for `resize`")
ansiblecube.run(emulation, ["resize"], extra_vars, secret_keys)
logger.step("Starting-up VM (second-time)")
with emulator.run(cancel_event) as emulation:
logger.step("Run ansiblecube phase I")
ansiblecube.run_phase_one(
emulation,
extra_vars,
secret_keys,
homepage=homepage_path,
logo=logo,
favicon=favicon,
css=css,
)
# wait for QEMU to release file (windows mostly)
time.sleep(10)
# mount image's 3rd partition on host
logger.stage("copy")
logger.step("Formating data partition on host")
format_data_partition(image_building_path, logger)
logger.step("Mounting data partition on host")
# copy contents from cache to mount point
try:
mount_point, device = mount_data_partition(image_building_path,
logger)
logger.step("Processing downloaded content onto data partition")
expanded_total_size = sum([c["expanded_size"] for c in downloads])
processed = 0
for category, content_dl_cb, content_run_cb, cb_kwargs in collection:
logger.step("Processing {cat}".format(cat=category))
content_run_cb(cache_folder=cache_folder,
mount_point=mount_point,
logger=logger,
**cb_kwargs)
# size of expanded files for this category (for progress)
processed += sum(
[c["expanded_size"] for c in content_dl_cb(**cb_kwargs)])
logger.progress(processed, expanded_total_size)
except Exception as exp:
try:
unmount_data_partition(mount_point, device, logger)
except NameError:
pass # if mount_point or device are not defined
raise exp
time.sleep(10)
# unmount partition
logger.step("Unmounting data partition")
unmount_data_partition(mount_point, device, logger)
time.sleep(10)
# rerun emulation for discovery
logger.stage("move")
logger.step("Starting-up VM (third-time)")
with emulator.run(cancel_event) as emulation:
logger.step("Run ansiblecube phase II")
ansiblecube.run_phase_two(emulation, extra_vars, secret_keys)
if shrink:
logger.step("Shrink size of physical image file")
# calculate physical size of image
required_image_size = get_required_image_size(collection)
if required_image_size + ONE_GB >= size:
# less than 1GB difference, don't bother
pass
else:
# set physical size to required + margin
physical_size = math.ceil(
required_image_size / ONE_GB) * ONE_GB
emulator.resize_image(physical_size, shrink=True)
# wait for QEMU to release file (windows mostly)
logger.succ("Image creation successful.")
time.sleep(20)
except Exception as e:
logger.failed(str(e))
# display traceback on logger
logger.std("\n--- Exception Trace ---\n{exp}\n---".format(
exp=traceback.format_exc()))
# Set final image filename
if os.path.isfile(image_building_path):
os.rename(image_building_path, image_error_path)
error = e
else:
try:
# Set final image filename
tries = 0
while True:
try:
os.rename(image_building_path, image_final_path)
except Exception as exp:
logger.err(exp)
tries += 1
if tries > 3:
raise exp
time.sleep(5 * tries)
continue
else:
logger.std(
"Renamed image file to {}".format(image_final_path))
break
# Write image to SD Card
if sd_card:
logger.stage("write")
logger.step("Writting image to SD-card ({})".format(sd_card))
try:
etcher_writer = EtcherWriterThread(args=(image_final_path,
sd_card, logger))
cancel_event.register_thread(thread=etcher_writer)
etcher_writer.start()
etcher_writer.join(timeout=2) # make sure it started
while etcher_writer.is_alive():
pass
logger.std("not alive")
etcher_writer.join(timeout=2)
cancel_event.unregister_thread()
if etcher_writer.exp is not None:
raise etcher_writer.exp
logger.std("Done writing and verifying.")
time.sleep(5)
except Exception:
logger.succ("Image created successfuly.")
logger.err(
"Writing or verification of Image to your SD-card failed.\n"
"Please use a third party tool to flash your image "
"onto your SD-card. See File menu for links to Etcher."
)
raise Exception("Failed to write Image to SD-card")
except Exception as e:
logger.failed(str(e))
# display traceback on logger
logger.std("\n--- Exception Trace ---\n{exp}\n---".format(
exp=traceback.format_exc()))
error = e
else:
logger.complete()
error = None
finally:
logger.std("Restoring system sleep policy")
restore_sleep_policy(sleep_ref, logger)
if sys.platform == "linux" and loop_dev and previous_loop_mode:
logger.step("Restoring loop device ({}) mode".format(loop_dev))
restore_mode(loop_dev, previous_loop_mode, logger)
# display durations summary
logger.summary()
if done_callback:
done_callback(error)
return error
def run_installation_button_clicked(self, button):
all_valid = True
project_name = self.component.project_name_entry.get_text()
allowed_chars = set(string.ascii_uppercase + string.ascii_lowercase +
string.digits + '-')
condition = len(project_name) >= 1 and len(project_name) <= 64 and set(
project_name) <= allowed_chars
validate_label(self.component.project_name_label, condition)
self.component.project_name_constraints_revealer.set_reveal_child(
not condition)
all_valid = all_valid and condition
language_id = self.component.language_combobox.get_active()
language = data.ideascube_languages[language_id][0]
condition = language_id != -1
validate_label(self.component.language_label, condition)
all_valid = all_valid and condition
timezone_id = self.component.timezone_combobox.get_active()
timezone = self.component.timezone_tree_store[timezone_id][0]
condition = timezone_id != -1
validate_label(self.component.timezone_label, condition)
all_valid = all_valid and condition
if self.component.wifi_password_switch.get_state():
wifi_pwd = None
condition = True
else:
wifi_pwd = self.component.wifi_password_entry.get_text()
condition = len(wifi_pwd) <= 31 and set(
wifi_pwd) <= set(string.ascii_uppercase +
string.ascii_lowercase + string.digits)
self.component.wifi_password_constraints_revealer.set_reveal_child(
not condition)
validate_label(self.component.wifi_password_label, condition)
all_valid = all_valid and condition
if not self.component.admin_account_switch.get_state():
admin_account = None
login_condition = True
pwd_condition = True
else:
admin_account = {
"login": self.component.admin_account_login_entry.get_text(),
"pwd": self.component.admin_account_pwd_entry.get_text(),
}
login_condition = len(admin_account["login"]) <= 31 and set(
admin_account["login"]) <= set(string.ascii_uppercase +
string.ascii_lowercase +
string.digits)
pwd_condition = len(admin_account["pwd"]) <= 31 and set(
admin_account["pwd"]) <= set(string.ascii_uppercase +
string.ascii_lowercase +
string.digits)
self.component.admin_account_login_constraints_revealer.set_reveal_child(
not login_condition)
self.component.admin_account_pwd_constraints_revealer.set_reveal_child(
not pwd_condition)
validate_label(self.component.admin_account_login_label,
login_condition)
validate_label(self.component.admin_account_pwd_label, pwd_condition)
all_valid = all_valid and pwd_condition and login_condition
zim_install = []
for zim in self.component.zim_list_store:
if zim[8]:
zim_install.append(zim[0])
output_size = self.get_output_size()
if self.component.output_stack.get_visible_child_name() == "sd_card":
sd_card_id = self.component.sd_card_combobox.get_active()
condition = sd_card_id != -1
validate_label(self.component.sd_card_label, condition)
all_valid = all_valid and condition
if sd_card_id == -1:
sd_card = None
else:
device_index = sd_card_info.get_device_index()
sd_card = self.component.sd_card_list_store[sd_card_id][
device_index]
else:
sd_card = None
condition = output_size > 0
validate_label(self.component.size_label, condition)
all_valid = all_valid and condition
condition = self.update_free_space() >= 0
validate_label(self.component.free_space_name_label, condition)
all_valid = all_valid and condition
kalite_active_langs = [
lang for lang, button in self.iter_kalite_check_button()
if button.get_active()
]
if len(kalite_active_langs) != 0:
kalite = kalite_active_langs
else:
kalite = None
wikifundi_active_langs = [
lang for lang, button in self.iter_wikifundi_check_button()
if button.get_active()
]
if len(wikifundi_active_langs) != 0:
wikifundi = wikifundi_active_langs
else:
wikifundi = None
aflatoun = self.component.aflatoun_switch.get_active()
edupi = self.component.edupi_switch.get_active()
logo = self.component.logo_chooser.get_filename()
favicon = self.component.favicon_chooser.get_filename()
css = self.component.css_chooser.get_filename()
build_dir = self.component.build_path_chooser.get_filename()
condition = build_dir != None
validate_label(self.component.build_path_chooser_label, condition)
all_valid = all_valid and condition
# Check if there is enough space in build_dir to build image
if build_dir != None:
free_space = get_free_space_in_dir(build_dir)
remaining_space = free_space - output_size
if remaining_space < 0:
self.component.space_error_image_location_label.set_text(
build_dir)
self.component.space_error_total_space_required_label.set_text(
human_readable_size(output_size))
self.component.space_error_space_available_label.set_text(
human_readable_size(free_space))
self.component.space_error_space_missing_label.set_text(
human_readable_size(-remaining_space))
self.component.space_error_window.show()
all_valid = False
if all_valid:
def target():
run_installation(name=project_name,
timezone=timezone,
language=language,
wifi_pwd=wifi_pwd,
kalite=kalite,
wikifundi=wikifundi,
aflatoun=aflatoun,
edupi=edupi,
zim_install=zim_install,
size=output_size,
logger=self.logger,
cancel_event=self.cancel_event,
sd_card=sd_card,
logo=logo,
favicon=favicon,
css=css,
build_dir=build_dir,
admin_account=admin_account,
done_callback=lambda error: GLib.idle_add(
self.installation_done, error))
self.component.window.hide()
self.reset_run_window()
self.component.run_window.show()
threading.Thread(target=target, daemon=True).start()
def __init__(self, catalog):
self.catalog = catalog
builder = Gtk.Builder()
builder.add_from_file(data.ui_glade)
self.component = Component(builder)
self.cancel_event = CancelEvent()
self.logger = Logger(self.component.run_text_view.get_buffer(),
self.component.run_step_label)
# main window
self.component.window.connect("delete-event", Gtk.main_quit)
# gtk file filters (macOS fix)
self.component.favicon_filter.set_name("Favicon (ICO, PNG)") # opt
self.component.favicon_filter.add_pattern("*.png")
self.component.favicon_filter.add_pattern("*.ico")
self.component.favicon_chooser.add_filter(
self.component.favicon_filter)
self.component.logo_filter.set_name("Logo (PNG)") # opt
self.component.logo_filter.add_pattern("*.png")
self.component.logo_chooser.add_filter(self.component.logo_filter)
self.component.css_filter.set_name("CSS File") # opt
self.component.css_filter.add_pattern("*.css")
self.component.css_chooser.add_filter(self.component.css_filter)
# menu bar
self.component.menu_quit.connect(
"activate", lambda widget: self.component.window.close())
self.component.menu_about.connect("activate", self.activate_menu_about)
self.component.menu_load_config.connect("activate",
self.activate_menu_config,
False)
self.component.menu_save_config.connect("activate",
self.activate_menu_config,
True)
# wifi password
self.component.wifi_password_switch.connect(
"notify::active", lambda switch, state: self.component.
wifi_password_revealer.set_reveal_child(not switch.get_active()))
# admin account
self.component.admin_account_switch.connect(
"notify::active", lambda switch, state: self.component.
admin_account_revealer.set_reveal_child(switch.get_active()))
# ideascube language
for code, language in data.ideascube_languages:
self.component.language_tree_store.append([code, language])
renderer = Gtk.CellRendererText()
self.component.language_combobox.pack_start(renderer, True)
self.component.language_combobox.add_attribute(renderer, "text", 1)
index = -1
for i, (code, language) in enumerate(data.ideascube_languages):
if code == 'en':
index = i
self.component.language_combobox.set_active(index)
# timezone
default_id = -1
local_tz = tzlocal.get_localzone()
for id, timezone in enumerate(pytz.common_timezones):
if timezone == "UTC" and default_id == -1:
default_id = id
if pytz.timezone(timezone) == local_tz:
default_id = id
self.component.timezone_tree_store.append(None, [timezone])
renderer = Gtk.CellRendererText()
self.component.timezone_combobox.pack_start(renderer, True)
self.component.timezone_combobox.add_attribute(renderer, "text", 0)
self.component.timezone_combobox.set_active(default_id)
# output
self.component.sd_card_combobox.connect(
"changed", lambda _: self.update_free_space())
self.component.sd_card_refresh_button.connect(
"clicked", self.sd_card_refresh_button_clicked)
self.component.output_stack.connect(
"notify::visible-child",
lambda switch, state: self.update_free_space())
self.component.size_entry.connect("changed",
lambda _: self.update_free_space())
types = [info["typ"] for info in sd_card_info.informations]
self.component.sd_card_list_store = Gtk.ListStore(*types)
self.component.sd_card_combobox.set_model(
self.component.sd_card_list_store)
for counter in range(0, sd_card_info.visible_informations):
cell_renderer = Gtk.CellRendererText()
self.component.sd_card_combobox.pack_start(cell_renderer, True)
self.component.sd_card_combobox.add_attribute(
cell_renderer, "text", counter)
# about dialog
self.component.about_dialog.set_logo(
GdkPixbuf.Pixbuf.new_from_file_at_scale(data.pibox_logo, 200, -1,
True))
# done window
self.component.done_window_ok_button.connect(
"clicked", lambda widget: self.component.done_window.hide())
self.component.done_window.connect("delete-event", hide_on_delete)
# space error window
self.component.space_error_window_ok_button.connect(
"clicked", self.space_error_window_ok_button_clicked)
self.component.space_error_window.connect("delete-event",
hide_on_delete)
# run window
self.component.run_installation_button.connect(
"clicked", self.run_installation_button_clicked)
self.component.run_window.connect("delete-event",
self.run_window_delete_event)
self.component.run_text_view.get_buffer().connect(
"modified-changed", self.run_text_view_scroll_down)
self.component.run_quit_button.connect("clicked",
self.run_quit_button_clicked)
self.component.run_abort_button.connect("clicked",
self.run_abort_button_clicked)
self.component.run_copy_log_to_clipboard_button.connect(
"clicked", self.run_copy_log_to_clipboard_button_clicked)
self.component.run_new_install_button.connect(
"clicked", self.run_new_install_button_clicked)
# zim content
self.component.zim_choose_content_button.connect(
"clicked", self.zim_choose_content_button_clicked)
self.component.zim_list_store = Gtk.ListStore(
str, # key
str, # name
str, # url
str, # description
str, # formatted_size
object, # languages
str, # type
str, # version
bool, # selected
str, # size
bool, # its language is selected
Gdk.RGBA # background color
)
self.component.zim_list_store.set_sort_column_id(
1, Gtk.SortType.ASCENDING)
all_languages = set()
for one_catalog in catalog:
for (key, value) in one_catalog["all"].items():
name = value["name"]
url = value["url"]
description = value.get("description") or "none"
# We double indicated size because in ideascube throught ansiblecube
# will first download the zip file and then extract the content
# TODO: an improvment would be to delete zip file after extraction and
# compute a temporar space needed that is max of all installed size
size = str(value["size"] * 2)
languages_iso = (value.get("language")
or "Unkown language").split(",")
languages = set(
map(lambda l: langcodes.Language.get(l).language_name(),
languages_iso))
typ = value["type"]
version = str(value["version"])
formatted_size = human_readable_size(int(size))
self.component.zim_list_store.append([
key, name, url, description, formatted_size, languages,
typ, version, False, size, True, VALID_RGBA
])
all_languages |= languages
self.component.zim_language_list_store = Gtk.ListStore(str)
self.component.zim_language_list_store.set_sort_column_id(
0, Gtk.SortType.ASCENDING)
for language in all_languages:
self.component.zim_language_list_store.append([language])
# zim window
self.component.zim_window_done_button.connect(
"clicked", self.zim_done_button_clicked)
self.component.zim_window.connect("delete-event", hide_on_delete)
self.component.zim_tree_view.connect("row-activated",
self.available_zim_clicked)
self.component.choosen_zim_tree_view.connect("row-activated",
self.choosen_zim_clicked)
## zim window available tree view
self.component.zim_tree_view.set_model(self.component.zim_list_store)
renderer_text = Gtk.CellRendererText()
column_text = Gtk.TreeViewColumn("Name", renderer_text, text=1)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Size", renderer_text, text=4)
self.component.zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Description", renderer_text, text=3)
self.component.zim_tree_view.append_column(column_text)
column_text.add_attribute(renderer_text, "cell_background_rgba", 11)
zim_filter = self.component.zim_list_store.filter_new()
zim_filter.set_visible_func(self.zim_filter_func)
self.component.zim_tree_view.set_model(zim_filter)
# zim window choosen tree view
self.component.choosen_zim_tree_view.set_model(
self.component.zim_list_store)
renderer_text = Gtk.CellRendererText()
column_text = Gtk.TreeViewColumn("Name", renderer_text, text=1)
self.component.choosen_zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Size", renderer_text, text=4)
self.component.choosen_zim_tree_view.append_column(column_text)
column_text = Gtk.TreeViewColumn("Description", renderer_text, text=3)
self.component.choosen_zim_tree_view.append_column(column_text)
choosen_zim_filter = self.component.zim_list_store.filter_new()
choosen_zim_filter.set_visible_func(self.choosen_zim_filter_func)
self.component.choosen_zim_tree_view.set_model(choosen_zim_filter)
# kalite
for lang, button in self.iter_kalite_check_button():
button.set_label("{} ({})".format(
button.get_label(),
human_readable_size(data.kalite_sizes[lang])))
button.connect("toggled", lambda button: self.update_free_space())
# wikifundi
for lang, button in self.iter_wikifundi_check_button():
button.set_label("{} ({})".format(
button.get_label(),
human_readable_size(data.wikifundi_sizes[lang])))
button.connect("toggled", lambda button: self.update_free_space())
# aflatoun
self.component.aflatoun_switch.connect(
"notify::active", lambda switch, state: self.update_free_space())
self.component.aflatoun_label.set_label("{} ({})".format(
self.component.aflatoun_label.get_label(),
human_readable_size(data.aflatoun_size)))
# edupi
self.component.edupi_switch.connect(
"notify::active", lambda switch, state: self.update_free_space())
self.component.edupi_label.set_label("{} ({})".format(
self.component.edupi_label.get_label(),
human_readable_size(data.edupi_size)))
# language tree view
renderer_text = Gtk.CellRendererText()
column_text = Gtk.TreeViewColumn("Language", renderer_text, text=0)
self.component.zim_language_tree_view.append_column(column_text)
self.component.zim_language_tree_view.get_selection().set_mode(
Gtk.SelectionMode(3))
self.component.zim_language_tree_view.set_model(
self.component.zim_language_list_store)
self.component.zim_language_tree_view.get_selection().select_all()
self.component.zim_language_tree_view.get_selection().connect(
"changed", self.zim_language_selection_changed)
self.refresh_disk_list()
self.update_free_space()
self.component.window.show()
def main(logger,
disk_size,
root_size,
build_folder,
qemu_ram,
image_fname=None):
# convert sizes to bytes and make sure those are usable
try:
root_size = int(root_size) * ONE_GB
disk_size = get_adjusted_image_size(int(disk_size) * ONE_GB)
if root_size < MIN_ROOT_SIZE:
raise ValueError("root partition must be at least {}".format(
human_readable_size(MIN_ROOT_SIZE, False)))
if root_size >= disk_size:
raise ValueError("root partition must be smaller than disk size")
except Exception as exp:
logger.err("Erroneous size option: {}".format(exp))
sys.exit(1)
logger.step("Starting master creation: {} ({} root)".format(
human_readable_size(disk_size, False),
human_readable_size(root_size, False)))
# default output file name
if image_fname is None:
image_fname = "hotspot-master_{date}.img".format(
date=datetime.datetime.now().strftime("%Y-%m-%d"))
image_fpath = os.path.join(build_folder, image_fname)
logger.step("starting with target: {}".format(image_fpath))
# download raspbian
logger.step("Retrieving raspbian image file")
raspbian_image = get_content("raspbian_image")
rf = download_content(raspbian_image, logger, build_folder)
if not rf.successful:
logger.err("Failed to download raspbian.\n{e}".format(e=rf.exception))
sys.exit(1)
elif rf.found:
logger.std("Reusing already downloaded raspbian ZIP file")
# extract raspbian and rename
logger.step("Extracting raspbian image from ZIP file")
unzip_file(
archive_fpath=rf.fpath,
src_fname=raspbian_image["name"].replace(".zip", ".img"),
build_folder=build_folder,
dest_fpath=image_fpath,
)
logger.std("Extraction complete: {p}".format(p=image_fpath))
if not os.path.exists(image_fpath):
raise IOError("image path does not exists: {}".format(image_fpath))
error = run_in_qemu(image_fpath, disk_size, root_size, logger,
CancelEvent(), qemu_ram)
if error:
logger.err("ERROR: unable to properly create image: {}".format(error))
sys.exit(1)
logger.std("SUCCESS! {} was built successfuly".format(image_fpath))
file_size = len(buf)
jpeg_size = jpegparser.read_jpeg(buf)['size']
wasted_size = file_size - jpeg_size
self.total_data += file_size
self.total_jpeg += jpeg_size
self.total_waste += wasted_size
args = parse_args()
scanner = JpegScanner()
try:
from tqdm import tqdm
except ImportError:
scanner.use_tqdm = False
if args.subcommand == 'dir':
scanner.scan_dir(args.dir)
elif args.subcommand == 'files':
scanner.scan_files(args.paths)
print()
print("total: %s" % (util.human_readable_size(scanner.total_data)))
print("real: %s (%.2f%%)" % (util.human_readable_size(
scanner.total_jpeg), scanner.total_jpeg / scanner.total_data * 100))
print("garbage: %s (%.2f%%)" % (util.human_readable_size(
scanner.total_waste), scanner.total_waste / scanner.total_data * 100))
"login": args.admin_account[0],
"pwd": args.admin_account[1]
}
else:
admin_account = {"login": "******", "pwd": "admin-password"}
# parse requested size
try:
args.size = humanfriendly.parse_size(args.size)
args.output_size = get_adjusted_image_size(
args.size) # adjust image size for HW
except Exception:
print("Unable to understand required size ({})".format(args.size))
sys.exit(1)
else:
args.human_size = human_readable_size(args.output_size, False)
# check arguments
(
valid_project_name,
valid_language,
valid_timezone,
valid_wifi_pwd,
valid_admin_login,
valid_admin_pwd,
) = check_user_inputs(
project_name=args.name,
language=args.language,
timezone=args.timezone,
wifi_pwd=args.wifi_pwd,
admin_login=admin_account["login"],
