def make_iso(yes: bool, source_disk: str, target_iso: str):
log.info(f'checking required files existence')
assert os.path.exists(os.path.abspath(os.path.join(target_iso, os.pardir)))
fdisk_output = shell_output(f'sudo fdisk -l {source_disk}')
print(fdisk_output)
block_size_line = [
line for line in fdisk_output.splitlines()
if line.startswith('Units: sectors of')
][0]
block_size_matcher = re.compile(r'= ([0-9]+) bytes$')
match = block_size_matcher.search(block_size_line)
assert match
block_size = int(match.group(1))
end_sector_line = fdisk_output.splitlines()[-1]
end_sector_matcher = re.compile(
f'^{source_disk}[a-z0-9]*\\s+[0-9]+\\s+([0-9]+)\\s+[0-9]+')
match = end_sector_matcher.search(end_sector_line)
assert match
end_sector = int(match.group(1))
log.info(f'block size: {block_size}')
log.info(f'end sector: {end_sector}')
confirm(
yes,
f'Attempting to dump partitions from {source_disk} to {target_iso}. Are you sure?'
)
log.info(f'Writing {source_disk} to {target_iso}')
wrap_shell(
f'sudo dd if={source_disk} of={target_iso} bs={block_size} count={end_sector} conv=noerror,sync status=progress'
)
wrap_shell('sync')
def create_os(dry: bool, yes: bool, disk: str, persistence: bool, boot_surplus: int, module: List[str], skip_fs: bool):
settings.DRY_RUN = dry
wrap_shell('lsblk')
confirm(yes, f'Attempting to create Wathmaker OS on {disk} disk, '
f'boot partition surplus: {boot_surplus} MiB, '
f'modules to be installed: {module}. '
f'Are you sure?')
creator.flash_disk(disk, persistence, boot_surplus, module, skip_fs)
def add_module(module: str, target_path):
log.info(f'Adding module {module}')
assert module in optional_modules
module_src_path = optional_modules[module]
assert os.path.exists(module_src_path), 'module src path not found'
if os.path.isdir(module_src_path):
dirname = os.path.basename(os.path.normpath(module_src_path))
log.info(
f'Copying module {module_src_path} to {target_path}/{dirname}')
wrap_shell(f'mkdir -p {target_path}/{dirname}')
wrap_shell(f'rsync -a {module_src_path}/ {target_path}/{dirname}/')
else:
assert module_src_path.endswith('.zip'), 'supporting .zip only'
log.info(f'Extracting module from {module_src_path} to {target_path}')
wrap_shell(f'unzip {module_src_path} -d {target_path}/')
wrap_shell(f'sync')
def replicate_os(source_disk: str, target_disk: str):
log.info(f'Cloning {source_disk} to {target_disk}...')
wrap_shell(
f'sudo dd if={source_disk} of={target_disk} bs=64K conv=noerror,sync status=progress'
)
wrap_shell('sync')
def flash_disk(disk: str, persistence: bool, boot_storage_surplus: int, modules: List[str], skip_fs: bool):
set_workdir(os.path.join(script_real_dir(), '..'))
log.info(f'checking required files existence')
assert os.path.exists('squash/filesystem.squashfs')
assert os.path.exists('content/boot-files')
assert os.path.exists('content/grub')
assert os.path.exists('modules/init')
# TODO unmount disk partitions
log.warn(f'writing to disk {disk}')
wrap_shell(f'df {disk}')
log.info('creating MBR')
wrap_shell(f'''sudo wipefs {disk}''')
wrap_shell(f'''sudo dd if=/dev/zero of={disk} seek=1 count=2047''')
wrap_shell(f'''
sudo parted --script {disk} \\
mklabel msdos
''')
log.info('calculating partitions size')
# depend on filesystem.squash size, expand by some surplus (for storage)
squashfs_size = os.path.getsize('squash/filesystem.squashfs')
boot_part_min_size = squashfs_size + dir_size('content/boot-files') + dir_size('content/grub')
boot_part_end_mib = boot_part_min_size / 1024 ** 2 + boot_storage_surplus
efi_part_end_mib = boot_part_end_mib + efi_part_size
persistence_part_end_mib = efi_part_end_mib + persistence_part_size
log.info(f'boot partition size: {boot_part_end_mib}MiB ({boot_storage_surplus}MiB surplus)')
log.info('creating partitions space')
if persistence:
wrap_shell(f'''
sudo parted --script {disk} \\
mkpart primary fat32 1MiB {boot_part_end_mib}MiB \\
set 1 lba on \\
set 1 boot on \\
mkpart primary fat32 {boot_part_end_mib}MiB {efi_part_end_mib}MiB \\
set 2 esp on \\
mkpart primary ext4 {efi_part_end_mib}MiB {persistence_part_end_mib}MiB \\
mkpart primary ext4 {persistence_part_end_mib}MiB 100%
''')
else:
wrap_shell(f'''
sudo parted --script {disk} \\
mkpart primary fat32 1MiB {boot_part_end_mib}MiB \\
set 1 lba on \\
set 1 boot on \\
mkpart primary fat32 {boot_part_end_mib}MiB {efi_part_end_mib}MiB \\
set 2 esp on \\
mkpart primary ext4 {efi_part_end_mib}MiB 100%
''')
wrap_shell('sync')
log.info('making boot partition filesystem')
wrap_shell(f'''sudo mkfs.fat -F32 {disk}1''')
log.info('making EFI partition filesystem')
wrap_shell(f'''sudo mkfs.fat -F32 {disk}2''')
if persistence:
log.info('making persistence partition filesystem')
wrap_shell(f'''sudo mkfs.ext4 -F {disk}3''')
log.info('making watchmodules partition filesystem')
wrap_shell(f'''sudo mkfs.ext4 -F {disk}4''')
else:
log.info('making watchmodules partition filesystem')
wrap_shell(f'''sudo mkfs.ext4 -F {disk}3''')
wrap_shell('sync')
log.info('setting partition names')
if persistence:
wrap_shell(f'''
sudo mlabel -i {disk}1 ::boot
sudo mlabel -i {disk}2 ::EFI
sudo e2label {disk}3 persistence
sudo e2label {disk}4 watchmodules
''')
else:
wrap_shell(f'''
sudo mlabel -i {disk}1 ::boot
sudo mlabel -i {disk}2 ::EFI
sudo e2label {disk}3 watchmodules
''')
wrap_shell('sync')
log.info('mounting partitions')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker''')
wrap_shell(f'''
sudo mkdir -p /mnt/watchmaker/boot
sudo mount {disk}1 /mnt/watchmaker/boot
''')
wrap_shell(f'''
sudo mkdir -p /mnt/watchmaker/efi
sudo mount {disk}2 /mnt/watchmaker/efi
''')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/watchmodules''')
if persistence:
wrap_shell(f'''
sudo mkdir -p /mnt/watchmaker/persistence
sudo mount {disk}3 /mnt/watchmaker/persistence
''')
wrap_shell(f'''sudo mount {disk}4 /mnt/watchmaker/watchmodules''')
else:
wrap_shell(f'''sudo mount {disk}3 /mnt/watchmaker/watchmodules''')
log.info('installing GRUB EFI bootloaders')
wrap_shell(f'''
sudo grub-install \\
--target=x86_64-efi \\
--efi-directory=/mnt/watchmaker/boot \\
--boot-directory=/mnt/watchmaker/boot/boot \\
--removable --recheck
''')
wrap_shell(f'''
sudo grub-install \\
--target=x86_64-efi \\
--efi-directory=/mnt/watchmaker/efi \\
--boot-directory=/mnt/watchmaker/boot/boot \\
--removable --recheck
''')
log.info('installing GRUB i386-pc bootloader')
wrap_shell(f'''
sudo grub-install \\
--target=i386-pc \\
--boot-directory=/mnt/watchmaker/boot/boot \\
--recheck \\
{disk}
''')
wrap_shell('sync')
log.info('Fixing GRUB EFI by replacing with Debian GRUB')
wrap_shell(f'''
sudo rm /mnt/watchmaker/efi/EFI/BOOT/*
sudo cp -r content/efi/* /mnt/watchmaker/efi/EFI/BOOT/
sudo rm /mnt/watchmaker/boot/EFI/BOOT/*
sudo cp -r content/efi/* /mnt/watchmaker/boot/EFI/BOOT/
sudo cp -r /mnt/watchmaker/efi/EFI/BOOT /mnt/watchmaker/efi/EFI/debian
sudo cp -r /mnt/watchmaker/boot/EFI/BOOT /mnt/watchmaker/boot/EFI/debian
sudo cp -r content/grub/x86_64-efi /mnt/watchmaker/boot/boot/grub/
''')
log.info('making EFI Microsoft workaround')
wrap_shell(f'''
sudo cp -r /mnt/watchmaker/efi/EFI/BOOT /mnt/watchmaker/efi/EFI/Microsoft
sudo cp -r /mnt/watchmaker/boot/EFI/BOOT /mnt/watchmaker/boot/EFI/Microsoft
''')
log.info('GRUB config')
wrap_shell(f'''
sudo cp content/grub/grub.cfg /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/background.png /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/font.pf2 /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/loopback.cfg /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/GRUB_FINDME /mnt/watchmaker/boot/
''')
log.info('Boot base files')
wrap_shell(f'''
sudo cp -r content/boot-files/[BOOT] /mnt/watchmaker/boot/
sudo cp -r content/boot-files/d-i /mnt/watchmaker/boot/
sudo cp -r content/boot-files/dists /mnt/watchmaker/boot/
sudo cp -r content/boot-files/live /mnt/watchmaker/boot/
sudo cp -r content/boot-files/pool /mnt/watchmaker/boot/
sudo cp -r content/boot-files/.disk /mnt/watchmaker/boot/
''')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/boot/storage''')
log.info('EFI base files')
wrap_shell(f'''
sudo cp -r content/boot-files/[BOOT] /mnt/watchmaker/efi/
sudo cp -r content/boot-files/d-i /mnt/watchmaker/efi/
sudo cp -r content/boot-files/dists /mnt/watchmaker/efi/
sudo cp -r content/boot-files/live /mnt/watchmaker/efi/
sudo cp -r content/boot-files/pool /mnt/watchmaker/efi/
sudo cp -r content/boot-files/.disk /mnt/watchmaker/efi/
''')
if persistence:
log.info('Persistence configuration')
wrap_shell(f'''sudo cp -r content/persistence/persistence.conf /mnt/watchmaker/persistence/''')
log.info('Copying squash filesystem')
if not skip_fs:
wrap_shell(f'''sudo cp squash/filesystem.squashfs /mnt/watchmaker/boot/live/''')
log.info('Adding init module')
wrap_shell(f'''sudo cp -r modules/init /mnt/watchmaker/watchmodules/''')
log.info('Adding dev module')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/watchmodules/dev''')
log.info('make watchmodules writable to non-root user')
wrap_shell(f'''sudo chown igrek /mnt/watchmaker/watchmodules -R''')
if modules:
log.info(f'Adding optional modules: {modules}')
target_path = '/mnt/watchmaker/watchmodules'
for module in modules:
install_module.add_module(module, target_path)
log.info('unmounting')
wrap_shell('sync')
wrap_shell(f'''sudo umount /mnt/watchmaker/boot''')
wrap_shell(f'''sudo umount /mnt/watchmaker/efi''')
wrap_shell(f'''sudo umount /mnt/watchmaker/watchmodules''')
if persistence:
wrap_shell(f'''sudo umount /mnt/watchmaker/persistence''')
wrap_shell('sync')
log.info('Success')
def replicate_os(dry: bool, yes: bool, source_disk: str, target_disk: str):
settings.DRY_RUN = dry
wrap_shell('lsblk -o NAME,TYPE,RM,RO,FSTYPE,SIZE,VENDOR,MODEL,LABEL,MOUNTPOINT')
confirm(yes, f'Attepmting to replicate OS from {source_disk} to {target_disk}. '
f'Are you sure?')
replicate.replicate_os(source_disk, target_disk)
def prebuild_tools(watchmaker_repo: str):
set_workdir(watchmaker_repo)
submodule_src_dir = f'{watchmaker_repo}/modules'
home = '/home/user'
log.info(f'checking required files existence')
assert os.path.exists(watchmaker_repo)
assert os.path.exists(submodule_src_dir)
assert os.path.exists(f'{submodule_src_dir}/lichking')
assert os.path.exists(f'{submodule_src_dir}/volumen')
assert os.geteuid() != 0, 'This script must not be run as root'
log.info('updating watchmaker tools itself')
wrap_shell(f'mkdir -p {home}/tools')
wrap_shell(f'rsync -a {watchmaker_repo}/watchmake/ {home}/tools/watchmake')
wrap_shell(f'rsync -a {watchmaker_repo}/scripts/ {home}/tools/scripts')
wrap_shell(f'cp {watchmaker_repo}/modules/music/tubular.wav {home}/Music/')
wrap_shell(f'cp {watchmaker_repo}/modules/music/tubular.mp3 {home}/Music/')
log.info('updating pip packages')
wrap_shell(f'sudo python3 -m pip install --upgrade nuclear')
wrap_shell(f'python3 -m pip install --upgrade diffs')
wrap_shell(f'python3 -m pip install --upgrade copymon')
wrap_shell(f'python3 -m pip install --upgrade regex-rename')
wrap_shell(f'python3 -m pip install --upgrade trimmer')
wrap_shell(f'python3 -m pip install --upgrade youtube-dl')
log.info('updating py-tools')
wrap_shell(f'rsync -a {submodule_src_dir}/lichking/ {home}/tools/lichking')
wrap_shell(f'rsync -a {submodule_src_dir}/volumen/ {home}/tools/volumen')
log.info('recreating links & autocompletion for tools')
wrap_shell(f'sudo rm -f /usr/bin/lichking')
wrap_shell(f'sudo rm -f /usr/bin/lich')
wrap_shell(f'sudo rm -f /usr/bin/king')
wrap_shell(f'sudo rm -f /usr/bin/volumen')
wrap_shell(f'sudo rm -f /usr/bin/watchmake')
wrap_shell(f'sudo rm -f /etc/bash_completion.d/cliglue_*')
wrap_shell(f'sudo rm -f /etc/bash_completion.d/nuclear_*')
wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash lichking')
wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash lich')
wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash king')
wrap_shell(f'{home}/tools/watchmake/watchmake.py --install-bash watchmake')
wrap_shell(f'{home}/tools/volumen/volumen.py --install-bash volumen')
wrap_shell(f'diffs --install-autocomplete')
wrap_shell(f'copymon --install-autocomplete')
wrap_shell(f'regex-rename --install-autocomplete')
wrap_shell(f'trimmer --install-autocomplete')
log.info('updating live dev repos')
wrap_shell(f'rm -rf {home}/dev-live')
wrap_shell(f'mkdir -p {home}/dev-live')
for repo_name, url in repo_remotes.items():
log.info(f'initializing live git repo {repo_name}')
repo_path = f'{home}/dev-live/{repo_name}'
wrap_shell(f'mkdir -p {repo_path}')
set_workdir(repo_path)
wrap_shell(f'git init')
wrap_shell(f'git remote add origin "{url}"')
set_workdir(watchmaker_repo)
log.info('clearing gradle cache')
wrap_shell(f'rm -rf {home}/.gradle/*')
log.info('clearing apt cache')
wrap_shell(f'sudo apt clean')
version_file = '/home/user/.osversion'
version_line = read_file(version_file).splitlines()[0]
version_matcher = re.compile(r'^v([0-9]+)\.([0-9]+)$')
match = version_matcher.match(version_line)
assert match
major_version = int(match.group(1))
minor_version = int(match.group(2)) + 1
new_version = f'v{major_version}.{minor_version}'
log.info(f'updating new OS version {new_version}')
save_file(version_file, new_version)
def resquash_os(storage_path: str, live_squash: str, exclude_file: str):
today = today_stamp()
squashfs_storage_path = f'{storage_path}/filesystem.squashfs'
tagged_squashfs_path = f'{storage_path}/filesystem-{today}.squashfs'
exclude_file_abs = os.path.abspath(exclude_file)
set_workdir('/')
# ensure mount points are mounted
log.info(f'checking mount points')
assert os.path.exists(
storage_path), f'storage path does not exist: {storage_path}'
assert os.path.exists(
live_squash), f'live squash file does not exist: {live_squash}'
assert os.path.exists(
exclude_file_abs), f'exclude file does not exist: {exclude_file_abs}'
log.info('removing old filesystem copy on storage')
wrap_shell(f'sudo rm -f {squashfs_storage_path}')
wrap_shell('sync')
log.info('squashing filesystem...')
wrap_shell(f'''
sudo mksquashfs \
/bin /boot /dev /etc /home /lib /lib64 /media /mnt /opt /proc /run /root /sbin /srv /sys /tmp /usr /var \
/initrd.img /initrd.img.old /vmlinuz /vmlinuz.old \
{squashfs_storage_path} \
-regex -ef {exclude_file_abs} \
-comp gzip -b 512k \
-keep-as-directory
''')
log.info(f'creating tagged copy: {tagged_squashfs_path}...')
wrap_shell(f'sudo cp {squashfs_storage_path} {tagged_squashfs_path}')
wrap_shell('sync')
log.info(f'cheking current squashfs size')
live_squash_mib = os.path.getsize(live_squash) / 1024**2
log.info(f'[!] Putting Live system at risk')
log.info(f'[!] removing current Live squashfs: {live_squash}')
wrap_shell(f'sudo rm -f {live_squash}')
log.info('[!] replacing with newest squashfs')
wrap_shell(
f'sudo rsync -ah --progress --no-perms --no-owner --no-group {squashfs_storage_path} {live_squash}'
)
wrap_shell('sync')
log.info(f'[!] Live system is functional again')
log.info(f'calculating checksum {squashfs_storage_path}')
cksum1 = checksum_file(squashfs_storage_path)
log.info(f'calculating checksum {live_squash}')
cksum2 = checksum_file(live_squash)
assert cksum1 == cksum2
log.info(f'checksums are valid')
tagged_squashfs_mib = os.path.getsize(tagged_squashfs_path) / 1024**2
squash_size_diff = tagged_squashfs_mib - live_squash_mib
log.info(
f'Success. '
f'Resquashed {live_squash}. '
f'Filesystem snaposhot dumped to {tagged_squashfs_path}',
size=f'{tagged_squashfs_mib}MiB',
size_diff=f'{squash_size_diff}MiB')