def make_iso(yes: bool, source_disk: str, target_iso: str): log.info(f'checking required files existence') assert os.path.exists(os.path.abspath(os.path.join(target_iso, os.pardir))) fdisk_output = shell_output(f'sudo fdisk -l {source_disk}') print(fdisk_output) block_size_line = [ line for line in fdisk_output.splitlines() if line.startswith('Units: sectors of') ][0] block_size_matcher = re.compile(r'= ([0-9]+) bytes$') match = block_size_matcher.search(block_size_line) assert match block_size = int(match.group(1)) end_sector_line = fdisk_output.splitlines()[-1] end_sector_matcher = re.compile( f'^{source_disk}[a-z0-9]*\\s+[0-9]+\\s+([0-9]+)\\s+[0-9]+') match = end_sector_matcher.search(end_sector_line) assert match end_sector = int(match.group(1)) log.info(f'block size: {block_size}') log.info(f'end sector: {end_sector}') confirm( yes, f'Attempting to dump partitions from {source_disk} to {target_iso}. Are you sure?' ) log.info(f'Writing {source_disk} to {target_iso}') wrap_shell( f'sudo dd if={source_disk} of={target_iso} bs={block_size} count={end_sector} conv=noerror,sync status=progress' ) wrap_shell('sync')
def create_os(dry: bool, yes: bool, disk: str, persistence: bool, boot_surplus: int, module: List[str], skip_fs: bool): settings.DRY_RUN = dry wrap_shell('lsblk') confirm(yes, f'Attempting to create Wathmaker OS on {disk} disk, ' f'boot partition surplus: {boot_surplus} MiB, ' f'modules to be installed: {module}. ' f'Are you sure?') creator.flash_disk(disk, persistence, boot_surplus, module, skip_fs)
def add_module(module: str, target_path): log.info(f'Adding module {module}') assert module in optional_modules module_src_path = optional_modules[module] assert os.path.exists(module_src_path), 'module src path not found' if os.path.isdir(module_src_path): dirname = os.path.basename(os.path.normpath(module_src_path)) log.info( f'Copying module {module_src_path} to {target_path}/{dirname}') wrap_shell(f'mkdir -p {target_path}/{dirname}') wrap_shell(f'rsync -a {module_src_path}/ {target_path}/{dirname}/') else: assert module_src_path.endswith('.zip'), 'supporting .zip only' log.info(f'Extracting module from {module_src_path} to {target_path}') wrap_shell(f'unzip {module_src_path} -d {target_path}/') wrap_shell(f'sync')
def replicate_os(source_disk: str, target_disk: str): log.info(f'Cloning {source_disk} to {target_disk}...') wrap_shell( f'sudo dd if={source_disk} of={target_disk} bs=64K conv=noerror,sync status=progress' ) wrap_shell('sync')
def flash_disk(disk: str, persistence: bool, boot_storage_surplus: int, modules: List[str], skip_fs: bool): set_workdir(os.path.join(script_real_dir(), '..')) log.info(f'checking required files existence') assert os.path.exists('squash/filesystem.squashfs') assert os.path.exists('content/boot-files') assert os.path.exists('content/grub') assert os.path.exists('modules/init') # TODO unmount disk partitions log.warn(f'writing to disk {disk}') wrap_shell(f'df {disk}') log.info('creating MBR') wrap_shell(f'''sudo wipefs {disk}''') wrap_shell(f'''sudo dd if=/dev/zero of={disk} seek=1 count=2047''') wrap_shell(f''' sudo parted --script {disk} \\ mklabel msdos ''') log.info('calculating partitions size') # depend on filesystem.squash size, expand by some surplus (for storage) squashfs_size = os.path.getsize('squash/filesystem.squashfs') boot_part_min_size = squashfs_size + dir_size('content/boot-files') + dir_size('content/grub') boot_part_end_mib = boot_part_min_size / 1024 ** 2 + boot_storage_surplus efi_part_end_mib = boot_part_end_mib + efi_part_size persistence_part_end_mib = efi_part_end_mib + persistence_part_size log.info(f'boot partition size: {boot_part_end_mib}MiB ({boot_storage_surplus}MiB surplus)') log.info('creating partitions space') if persistence: wrap_shell(f''' sudo parted --script {disk} \\ mkpart primary fat32 1MiB {boot_part_end_mib}MiB \\ set 1 lba on \\ set 1 boot on \\ mkpart primary fat32 {boot_part_end_mib}MiB {efi_part_end_mib}MiB \\ set 2 esp on \\ mkpart primary ext4 {efi_part_end_mib}MiB {persistence_part_end_mib}MiB \\ mkpart primary ext4 {persistence_part_end_mib}MiB 100% ''') else: wrap_shell(f''' sudo parted --script {disk} \\ mkpart primary fat32 1MiB {boot_part_end_mib}MiB \\ set 1 lba on \\ set 1 boot on \\ mkpart primary fat32 {boot_part_end_mib}MiB {efi_part_end_mib}MiB \\ set 2 esp on \\ mkpart primary ext4 {efi_part_end_mib}MiB 100% ''') wrap_shell('sync') log.info('making boot partition filesystem') wrap_shell(f'''sudo mkfs.fat -F32 {disk}1''') log.info('making EFI partition filesystem') wrap_shell(f'''sudo mkfs.fat -F32 {disk}2''') if persistence: log.info('making persistence partition filesystem') wrap_shell(f'''sudo mkfs.ext4 -F {disk}3''') log.info('making watchmodules partition filesystem') wrap_shell(f'''sudo mkfs.ext4 -F {disk}4''') else: log.info('making watchmodules partition filesystem') wrap_shell(f'''sudo mkfs.ext4 -F {disk}3''') wrap_shell('sync') log.info('setting partition names') if persistence: wrap_shell(f''' sudo mlabel -i {disk}1 ::boot sudo mlabel -i {disk}2 ::EFI sudo e2label {disk}3 persistence sudo e2label {disk}4 watchmodules ''') else: wrap_shell(f''' sudo mlabel -i {disk}1 ::boot sudo mlabel -i {disk}2 ::EFI sudo e2label {disk}3 watchmodules ''') wrap_shell('sync') log.info('mounting partitions') wrap_shell(f'''sudo mkdir -p /mnt/watchmaker''') wrap_shell(f''' sudo mkdir -p /mnt/watchmaker/boot sudo mount {disk}1 /mnt/watchmaker/boot ''') wrap_shell(f''' sudo mkdir -p /mnt/watchmaker/efi sudo mount {disk}2 /mnt/watchmaker/efi ''') wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/watchmodules''') if persistence: wrap_shell(f''' sudo mkdir -p /mnt/watchmaker/persistence sudo mount {disk}3 /mnt/watchmaker/persistence ''') wrap_shell(f'''sudo mount {disk}4 /mnt/watchmaker/watchmodules''') else: wrap_shell(f'''sudo mount {disk}3 /mnt/watchmaker/watchmodules''') log.info('installing GRUB EFI bootloaders') wrap_shell(f''' sudo grub-install \\ --target=x86_64-efi \\ --efi-directory=/mnt/watchmaker/boot \\ --boot-directory=/mnt/watchmaker/boot/boot \\ --removable --recheck ''') wrap_shell(f''' sudo grub-install \\ --target=x86_64-efi \\ --efi-directory=/mnt/watchmaker/efi \\ --boot-directory=/mnt/watchmaker/boot/boot \\ --removable --recheck ''') log.info('installing GRUB i386-pc bootloader') wrap_shell(f''' sudo grub-install \\ --target=i386-pc \\ --boot-directory=/mnt/watchmaker/boot/boot \\ --recheck \\ {disk} ''') wrap_shell('sync') log.info('Fixing GRUB EFI by replacing with Debian GRUB') wrap_shell(f''' sudo rm /mnt/watchmaker/efi/EFI/BOOT/* sudo cp -r content/efi/* /mnt/watchmaker/efi/EFI/BOOT/ sudo rm /mnt/watchmaker/boot/EFI/BOOT/* sudo cp -r content/efi/* /mnt/watchmaker/boot/EFI/BOOT/ sudo cp -r /mnt/watchmaker/efi/EFI/BOOT /mnt/watchmaker/efi/EFI/debian sudo cp -r /mnt/watchmaker/boot/EFI/BOOT /mnt/watchmaker/boot/EFI/debian sudo cp -r content/grub/x86_64-efi /mnt/watchmaker/boot/boot/grub/ ''') log.info('making EFI Microsoft workaround') wrap_shell(f''' sudo cp -r /mnt/watchmaker/efi/EFI/BOOT /mnt/watchmaker/efi/EFI/Microsoft sudo cp -r /mnt/watchmaker/boot/EFI/BOOT /mnt/watchmaker/boot/EFI/Microsoft ''') log.info('GRUB config') wrap_shell(f''' sudo cp content/grub/grub.cfg /mnt/watchmaker/boot/boot/grub/ sudo cp content/grub/background.png /mnt/watchmaker/boot/boot/grub/ sudo cp content/grub/font.pf2 /mnt/watchmaker/boot/boot/grub/ sudo cp content/grub/loopback.cfg /mnt/watchmaker/boot/boot/grub/ sudo cp content/grub/GRUB_FINDME /mnt/watchmaker/boot/ ''') log.info('Boot base files') wrap_shell(f''' sudo cp -r content/boot-files/[BOOT] /mnt/watchmaker/boot/ sudo cp -r content/boot-files/d-i /mnt/watchmaker/boot/ sudo cp -r content/boot-files/dists /mnt/watchmaker/boot/ sudo cp -r content/boot-files/live /mnt/watchmaker/boot/ sudo cp -r content/boot-files/pool /mnt/watchmaker/boot/ sudo cp -r content/boot-files/.disk /mnt/watchmaker/boot/ ''') wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/boot/storage''') log.info('EFI base files') wrap_shell(f''' sudo cp -r content/boot-files/[BOOT] /mnt/watchmaker/efi/ sudo cp -r content/boot-files/d-i /mnt/watchmaker/efi/ sudo cp -r content/boot-files/dists /mnt/watchmaker/efi/ sudo cp -r content/boot-files/live /mnt/watchmaker/efi/ sudo cp -r content/boot-files/pool /mnt/watchmaker/efi/ sudo cp -r content/boot-files/.disk /mnt/watchmaker/efi/ ''') if persistence: log.info('Persistence configuration') wrap_shell(f'''sudo cp -r content/persistence/persistence.conf /mnt/watchmaker/persistence/''') log.info('Copying squash filesystem') if not skip_fs: wrap_shell(f'''sudo cp squash/filesystem.squashfs /mnt/watchmaker/boot/live/''') log.info('Adding init module') wrap_shell(f'''sudo cp -r modules/init /mnt/watchmaker/watchmodules/''') log.info('Adding dev module') wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/watchmodules/dev''') log.info('make watchmodules writable to non-root user') wrap_shell(f'''sudo chown igrek /mnt/watchmaker/watchmodules -R''') if modules: log.info(f'Adding optional modules: {modules}') target_path = '/mnt/watchmaker/watchmodules' for module in modules: install_module.add_module(module, target_path) log.info('unmounting') wrap_shell('sync') wrap_shell(f'''sudo umount /mnt/watchmaker/boot''') wrap_shell(f'''sudo umount /mnt/watchmaker/efi''') wrap_shell(f'''sudo umount /mnt/watchmaker/watchmodules''') if persistence: wrap_shell(f'''sudo umount /mnt/watchmaker/persistence''') wrap_shell('sync') log.info('Success')
def replicate_os(dry: bool, yes: bool, source_disk: str, target_disk: str): settings.DRY_RUN = dry wrap_shell('lsblk -o NAME,TYPE,RM,RO,FSTYPE,SIZE,VENDOR,MODEL,LABEL,MOUNTPOINT') confirm(yes, f'Attepmting to replicate OS from {source_disk} to {target_disk}. ' f'Are you sure?') replicate.replicate_os(source_disk, target_disk)
def prebuild_tools(watchmaker_repo: str): set_workdir(watchmaker_repo) submodule_src_dir = f'{watchmaker_repo}/modules' home = '/home/user' log.info(f'checking required files existence') assert os.path.exists(watchmaker_repo) assert os.path.exists(submodule_src_dir) assert os.path.exists(f'{submodule_src_dir}/lichking') assert os.path.exists(f'{submodule_src_dir}/volumen') assert os.geteuid() != 0, 'This script must not be run as root' log.info('updating watchmaker tools itself') wrap_shell(f'mkdir -p {home}/tools') wrap_shell(f'rsync -a {watchmaker_repo}/watchmake/ {home}/tools/watchmake') wrap_shell(f'rsync -a {watchmaker_repo}/scripts/ {home}/tools/scripts') wrap_shell(f'cp {watchmaker_repo}/modules/music/tubular.wav {home}/Music/') wrap_shell(f'cp {watchmaker_repo}/modules/music/tubular.mp3 {home}/Music/') log.info('updating pip packages') wrap_shell(f'sudo python3 -m pip install --upgrade nuclear') wrap_shell(f'python3 -m pip install --upgrade diffs') wrap_shell(f'python3 -m pip install --upgrade copymon') wrap_shell(f'python3 -m pip install --upgrade regex-rename') wrap_shell(f'python3 -m pip install --upgrade trimmer') wrap_shell(f'python3 -m pip install --upgrade youtube-dl') log.info('updating py-tools') wrap_shell(f'rsync -a {submodule_src_dir}/lichking/ {home}/tools/lichking') wrap_shell(f'rsync -a {submodule_src_dir}/volumen/ {home}/tools/volumen') log.info('recreating links & autocompletion for tools') wrap_shell(f'sudo rm -f /usr/bin/lichking') wrap_shell(f'sudo rm -f /usr/bin/lich') wrap_shell(f'sudo rm -f /usr/bin/king') wrap_shell(f'sudo rm -f /usr/bin/volumen') wrap_shell(f'sudo rm -f /usr/bin/watchmake') wrap_shell(f'sudo rm -f /etc/bash_completion.d/cliglue_*') wrap_shell(f'sudo rm -f /etc/bash_completion.d/nuclear_*') wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash lichking') wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash lich') wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash king') wrap_shell(f'{home}/tools/watchmake/watchmake.py --install-bash watchmake') wrap_shell(f'{home}/tools/volumen/volumen.py --install-bash volumen') wrap_shell(f'diffs --install-autocomplete') wrap_shell(f'copymon --install-autocomplete') wrap_shell(f'regex-rename --install-autocomplete') wrap_shell(f'trimmer --install-autocomplete') log.info('updating live dev repos') wrap_shell(f'rm -rf {home}/dev-live') wrap_shell(f'mkdir -p {home}/dev-live') for repo_name, url in repo_remotes.items(): log.info(f'initializing live git repo {repo_name}') repo_path = f'{home}/dev-live/{repo_name}' wrap_shell(f'mkdir -p {repo_path}') set_workdir(repo_path) wrap_shell(f'git init') wrap_shell(f'git remote add origin "{url}"') set_workdir(watchmaker_repo) log.info('clearing gradle cache') wrap_shell(f'rm -rf {home}/.gradle/*') log.info('clearing apt cache') wrap_shell(f'sudo apt clean') version_file = '/home/user/.osversion' version_line = read_file(version_file).splitlines()[0] version_matcher = re.compile(r'^v([0-9]+)\.([0-9]+)$') match = version_matcher.match(version_line) assert match major_version = int(match.group(1)) minor_version = int(match.group(2)) + 1 new_version = f'v{major_version}.{minor_version}' log.info(f'updating new OS version {new_version}') save_file(version_file, new_version)
def resquash_os(storage_path: str, live_squash: str, exclude_file: str): today = today_stamp() squashfs_storage_path = f'{storage_path}/filesystem.squashfs' tagged_squashfs_path = f'{storage_path}/filesystem-{today}.squashfs' exclude_file_abs = os.path.abspath(exclude_file) set_workdir('/') # ensure mount points are mounted log.info(f'checking mount points') assert os.path.exists( storage_path), f'storage path does not exist: {storage_path}' assert os.path.exists( live_squash), f'live squash file does not exist: {live_squash}' assert os.path.exists( exclude_file_abs), f'exclude file does not exist: {exclude_file_abs}' log.info('removing old filesystem copy on storage') wrap_shell(f'sudo rm -f {squashfs_storage_path}') wrap_shell('sync') log.info('squashing filesystem...') wrap_shell(f''' sudo mksquashfs \ /bin /boot /dev /etc /home /lib /lib64 /media /mnt /opt /proc /run /root /sbin /srv /sys /tmp /usr /var \ /initrd.img /initrd.img.old /vmlinuz /vmlinuz.old \ {squashfs_storage_path} \ -regex -ef {exclude_file_abs} \ -comp gzip -b 512k \ -keep-as-directory ''') log.info(f'creating tagged copy: {tagged_squashfs_path}...') wrap_shell(f'sudo cp {squashfs_storage_path} {tagged_squashfs_path}') wrap_shell('sync') log.info(f'cheking current squashfs size') live_squash_mib = os.path.getsize(live_squash) / 1024**2 log.info(f'[!] Putting Live system at risk') log.info(f'[!] removing current Live squashfs: {live_squash}') wrap_shell(f'sudo rm -f {live_squash}') log.info('[!] replacing with newest squashfs') wrap_shell( f'sudo rsync -ah --progress --no-perms --no-owner --no-group {squashfs_storage_path} {live_squash}' ) wrap_shell('sync') log.info(f'[!] Live system is functional again') log.info(f'calculating checksum {squashfs_storage_path}') cksum1 = checksum_file(squashfs_storage_path) log.info(f'calculating checksum {live_squash}') cksum2 = checksum_file(live_squash) assert cksum1 == cksum2 log.info(f'checksums are valid') tagged_squashfs_mib = os.path.getsize(tagged_squashfs_path) / 1024**2 squash_size_diff = tagged_squashfs_mib - live_squash_mib log.info( f'Success. ' f'Resquashed {live_squash}. ' f'Filesystem snaposhot dumped to {tagged_squashfs_path}', size=f'{tagged_squashfs_mib}MiB', size_diff=f'{squash_size_diff}MiB')