def download_from_youtube(url: str) -> str:
with wrap_context('downloading from youtube', url=url):
log.info('downloading from youtube...', url=url)
uid = str(uuid.uuid4())
filename = f'trimmer_dl_{uid}'
ydl_opts = {
'format': 'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'outtmpl': f'{filename}.%(ext)s'
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
retcode = ydl.download([url])
assert retcode == 0
full_filename = f'{filename}.mp3'
assert os.path.isfile(full_filename)
log.info('song downloaded', tmpfile=full_filename)
return full_filename
def volume_show():
master_volume = read_master_volume()
icon_name = get_notification_icon(master_volume)
summary = 'Volume'
body = f'{master_volume:d}%'
log.info(f'Volume: {body}')
show_notification(icon_name, summary, body)
def log(self, verbose: int):
ctx = {}
if verbose >= 2:
ctx['headers'] = self.headers
if self.content:
ctx['content'] = '\n' + self.content.decode('utf-8')
log.info(f'< Incoming {self.method} {self.path}', **ctx)
def read_mp3_artist_title(mp3_file: str) -> Tuple[str, str]:
with wrap_context('extracting mp3 metadata', mp3_file=mp3_file):
tag_artist, tag_title = read_mp3_tags(mp3_file)
file_artist, file_title = extract_filename_artist_title(mp3_file)
artist = tag_artist or file_artist
title = tag_title or file_title
log.info('metadata inferred', artist=artist, title=title)
return artist, title
def tag_mp3(mp3_file: str, artist: str, title: str):
with wrap_context('tagging mp3', artist=artist, title=title, mp3_file=mp3_file):
log.info('tagging mp3...', artist=artist, title=title)
audiofile = eyed3.load(mp3_file)
audiofile.tag.artist = artist
audiofile.tag.title = title
audiofile.tag.save(version=ID3_V1_1)
audiofile.tag.save(version=ID3_V2_4)
def read_mp3_tags(mp3_file: str) -> Tuple[str, str]:
with wrap_context('reading mp3 tags'):
audiofile = eyed3.load(mp3_file)
if audiofile is None or audiofile.tag is None:
log.warn('no IDv3 tags read', mp3_file=mp3_file)
return '', ''
artist = _oremptystr(audiofile.tag.artist).strip()
title = _oremptystr(audiofile.tag.title).strip()
log.info('IDv3 tags read', mp3_file=mp3_file, artist=artist, title=title)
return artist, title
def make_iso(yes: bool, source_disk: str, target_iso: str):
log.info(f'checking required files existence')
assert os.path.exists(os.path.abspath(os.path.join(target_iso, os.pardir)))
fdisk_output = shell_output(f'sudo fdisk -l {source_disk}')
print(fdisk_output)
block_size_line = [
line for line in fdisk_output.splitlines()
if line.startswith('Units: sectors of')
][0]
block_size_matcher = re.compile(r'= ([0-9]+) bytes$')
match = block_size_matcher.search(block_size_line)
assert match
block_size = int(match.group(1))
end_sector_line = fdisk_output.splitlines()[-1]
end_sector_matcher = re.compile(
f'^{source_disk}[a-z0-9]*\\s+[0-9]+\\s+([0-9]+)\\s+[0-9]+')
match = end_sector_matcher.search(end_sector_line)
assert match
end_sector = int(match.group(1))
log.info(f'block size: {block_size}')
log.info(f'end sector: {end_sector}')
confirm(
yes,
f'Attempting to dump partitions from {source_disk} to {target_iso}. Are you sure?'
)
log.info(f'Writing {source_disk} to {target_iso}')
wrap_shell(
f'sudo dd if={source_disk} of={target_iso} bs={block_size} count={end_sector} conv=noerror,sync status=progress'
)
wrap_shell('sync')
def rename_song(mp3_file: str, artist: str, title: str) -> str:
with wrap_context('renaming song',
artist=artist,
title=title,
mp3_file=mp3_file):
dirname, filename = os.path.split(mp3_file)
if artist.strip():
new_filename = f'{artist.strip()} - {title.strip()}.mp3'
else:
new_filename = f'{title.strip()}.mp3'
new_path = Path(dirname) / new_filename
os.rename(mp3_file, new_path)
log.info('song renamed', new_name=new_path)
return new_path
def trim_from_source(source: str, artist: Optional[str], title: Optional[str],
no_trim: bool, no_fade: bool, no_normalize: bool, no_rename: bool,
trim_start: Optional[float], trim_end: Optional[float], gain: Optional[float],
output: Optional[str]):
with log_error():
if source_is_url(source):
log.info('source recognized as url', source=source)
trim_url(source, artist, title, no_trim, no_fade, no_normalize,
trim_start, trim_end, gain, output)
elif source_is_mp3(source):
log.info('source recognized as mp3 file', source=source)
trim_mp3(source, artist, title, no_trim, no_fade, no_normalize, no_rename,
trim_start, trim_end, gain, output)
else:
raise RuntimeError(f'unrecognized source: {source}')
def _init_request_cache(self) -> Dict[int, CacheEntry]:
if self.config.record_file and os.path.isfile(self.config.record_file):
txt = Path(self.config.record_file).read_text()
if not txt:
return {}
entries = json.loads(txt)
loaded_cache = {}
for entry in entries:
parsed_entry = CacheEntry.from_json(entry)
request_hash = self._request_hash(parsed_entry.request)
loaded_cache[request_hash] = parsed_entry
conflicts = len(entries) - len(loaded_cache)
log.info(f'Loaded cached request-response entries',
record_file=self.config.record_file,
loaded=len(loaded_cache),
conflicts=conflicts)
return loaded_cache
return {}
def load_extensions(extension_path: str) -> Extensions:
if not extension_path:
return Extensions()
ext = Extensions()
ext_module = SourceFileLoader('xman.extension',
extension_path).load_module()
ext_names = [field.name for field in fields(Extensions)]
loaded = []
for ext_name in ext_names:
if hasattr(ext_module, ext_name):
ext_value = getattr(ext_module, ext_name)
if ext_value:
setattr(ext, ext_name, ext_value)
loaded.append(ext_name)
log.info('Loaded extensions', file=extension_path, extensions=loaded)
return ext
def add_modules(yes: bool, modules: List[str]):
set_workdir(os.path.join(script_real_dir(), '..'))
log.info(f'checking required files existence')
assert os.path.exists('squash/filesystem.squashfs')
assert os.path.exists('content/boot-files')
assert os.path.exists('content/grub')
assert os.path.exists('modules/init')
target_path = find_usb_data_partition()
assert os.path.exists(target_path), 'module target path not found'
confirm(
yes,
f'Attepmting to install modules {modules} to {target_path}. Are you sure?'
)
log.info(f'Adding optional modules: {modules}')
for module in modules:
add_module(module, target_path)
def trim_mp3(mp3_file: str, user_artist: Optional[str], user_title: Optional[str],
no_trim: bool, no_fade: bool, no_normalize: bool, no_rename: bool,
trim_start: Optional[float], trim_end: Optional[float], gain: Optional[float], output: Optional[str]):
with wrap_context('mp3 song'):
assert os.path.isfile(mp3_file), 'input file should exist'
if output:
shutil.copyfile(mp3_file, output)
mp3_file = output
tag_artist, tag_title = read_mp3_artist_title(mp3_file)
artist = user_artist or tag_artist or enter_or_default('Artist', default='')
title = user_title or tag_title or enter_or_default('Title', default='')
if not output and not no_rename:
mp3_file = rename_song(mp3_file, artist, title)
normalize_song(mp3_file, no_trim, no_fade, no_normalize, trim_start, trim_end, gain)
tag_mp3(mp3_file, artist, title)
log.info('song saved', mp3_file=mp3_file)
def setup_proxy(listen_port: int, listen_ssl: bool, dst_url: str, record: bool,
record_file: str, replay: bool, replay_throttle: bool,
replay_clear_cache: bool, replay_clear_cache_seconds: int,
config: str, verbose: int):
with logerr():
with wrap_context('initialization'):
extensions = load_extensions(config)
_config = Config(
listen_port=listen_port,
listen_ssl=listen_ssl,
dst_url=dst_url,
record=record,
record_file=record_file,
replay=replay,
replay_throttle=replay_throttle,
replay_clear_cache=replay_clear_cache,
replay_clear_cache_seconds=replay_clear_cache_seconds,
verbose=verbose,
)
if extensions.override_config:
extensions.override_config(_config)
log.info('Configuration set', **asdict(_config))
RequestHandler.extensions = extensions
RequestHandler.config = _config
RequestHandler.cache = RequestCache(extensions, _config)
TCPServer.allow_reuse_address = True
httpd = TCPServer((_config.listen_addr, _config.listen_port),
RequestHandler)
if _config.listen_ssl:
httpd.socket = ssl.wrap_socket(httpd.socket,
certfile='./dev-cert.pem',
server_side=True)
log.info(
f'Listening on {_config.listen_scheme} port {_config.listen_port}...'
)
try:
httpd.serve_forever()
finally:
httpd.server_close()
def fetch_youtube_metadata(url: str) -> Tuple[str, str, str]:
with wrap_context('fetching title from youtube', url=url):
log.info('fetching metadata from youtube page...', url=url)
ydl_opts = {
'format': 'bestaudio/best',
'postprocessors': [{
'key': 'FFmpegExtractAudio',
'preferredcodec': 'mp3',
'preferredquality': '192',
}],
'outtmpl': '%(title)s.%(ext)s',
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
einfo = ydl.extract_info(url, download=False)
track = einfo.get('track')
artist = einfo.get('artist') or einfo.get('creator')
full_title = einfo.get('title') or einfo.get('alt_title')
log.info('youtube page metadata fetched', yt_title=full_title, artist=artist, track=track)
return artist, track, full_title
def bulk_rename(
pattern: str,
replacement_pattern: Optional[str],
testing: bool = True,
full: bool = False,
recursive: bool = False,
padding: int = 0,
) -> List[Match]:
"""
Rename (or match) multiple files at once
:param pattern: regex pattern to match filenames
:param replacement: replacement regex pattern for renamed files.
Use \\1 syntax to make use of matched groups
:param testing: True - just testing replacement pattern, False - do actual renaming files
:param full: whether to enforce matching full filename against pattern
:param recursive: whether to search directories recursively
:param padding: applies padding with zeros with given length on matched numerical groups
"""
log.debug('matching regex pattern',
pattern=pattern,
replacement=replacement_pattern,
testing_mode=testing,
full_match=full,
recursive=recursive,
padding=padding)
matches: List[Match] = match_files(Path(), pattern, replacement_pattern,
recursive, full, padding)
for match in matches:
match.log_info(testing)
if replacement_pattern:
find_duplicates(matches)
if testing:
if matches:
log.info('files matched', count=len(matches))
else:
log.info('no files matched', count=len(matches))
elif replacement_pattern:
rename_matches(matches)
if matches:
log.info('files renamed', count=len(matches))
else:
log.info('no files renamed', count=len(matches))
else:
raise RuntimeError('replacement pattern is required for renaming')
return matches
def log_info(self, testing: bool):
group_kwargs = {f'group_{idx}': group for idx, group in self.groups.items()}
if self.name_to is None:
log.info('matched file', file=self.name_from, **group_kwargs)
else:
if testing:
log.info('matched file', **{'from': self.name_from, 'to': self.name_to}, **group_kwargs)
else:
log.info('renaming file', **{'from': self.name_from, 'to': self.name_to})
def add_module(module: str, target_path):
log.info(f'Adding module {module}')
assert module in optional_modules
module_src_path = optional_modules[module]
assert os.path.exists(module_src_path), 'module src path not found'
if os.path.isdir(module_src_path):
dirname = os.path.basename(os.path.normpath(module_src_path))
log.info(
f'Copying module {module_src_path} to {target_path}/{dirname}')
wrap_shell(f'mkdir -p {target_path}/{dirname}')
wrap_shell(f'rsync -a {module_src_path}/ {target_path}/{dirname}/')
else:
assert module_src_path.endswith('.zip'), 'supporting .zip only'
log.info(f'Extracting module from {module_src_path} to {target_path}')
wrap_shell(f'unzip {module_src_path} -d {target_path}/')
wrap_shell(f'sync')
def trim_url(url: str, user_artist: Optional[str], user_title: Optional[str],
no_trim: bool, no_fade: bool, no_normalize: bool,
trim_start: Optional[float], trim_end: Optional[float], gain: Optional[float], output: Optional[str]):
with wrap_context('url song'):
yt_artist, yt_title = extract_youtube_artist_title(url)
log.info('artist & title extracted from youtube page', artist=yt_artist, title=yt_title)
artist = user_artist or enter_or_default('Artist', default=yt_artist)
title = user_title or enter_or_default('Title', default=yt_title)
log.info('song name set', name=f'{artist} - {title}')
mp3_file = download_from_youtube(url)
if output:
mp3_file = rename_output_song(mp3_file, output)
else:
mp3_file = rename_song(mp3_file, artist, title)
normalize_song(mp3_file, no_trim, no_fade, no_normalize, trim_start, trim_end, gain)
tag_mp3(mp3_file, artist, title)
log.info('song saved', mp3_file=mp3_file)
def rename_output_song(mp3_file: str, output: str) -> str:
with wrap_context('renaming song', mp3_file=mp3_file, output=output):
os.rename(mp3_file, output)
log.info('song renamed', new_name=output)
return output
def prebuild_tools(watchmaker_repo: str):
set_workdir(watchmaker_repo)
submodule_src_dir = f'{watchmaker_repo}/modules'
home = '/home/user'
log.info(f'checking required files existence')
assert os.path.exists(watchmaker_repo)
assert os.path.exists(submodule_src_dir)
assert os.path.exists(f'{submodule_src_dir}/lichking')
assert os.path.exists(f'{submodule_src_dir}/volumen')
assert os.geteuid() != 0, 'This script must not be run as root'
log.info('updating watchmaker tools itself')
wrap_shell(f'mkdir -p {home}/tools')
wrap_shell(f'rsync -a {watchmaker_repo}/watchmake/ {home}/tools/watchmake')
wrap_shell(f'rsync -a {watchmaker_repo}/scripts/ {home}/tools/scripts')
wrap_shell(f'cp {watchmaker_repo}/modules/music/tubular.wav {home}/Music/')
wrap_shell(f'cp {watchmaker_repo}/modules/music/tubular.mp3 {home}/Music/')
log.info('updating pip packages')
wrap_shell(f'sudo python3 -m pip install --upgrade nuclear')
wrap_shell(f'python3 -m pip install --upgrade diffs')
wrap_shell(f'python3 -m pip install --upgrade copymon')
wrap_shell(f'python3 -m pip install --upgrade regex-rename')
wrap_shell(f'python3 -m pip install --upgrade trimmer')
wrap_shell(f'python3 -m pip install --upgrade youtube-dl')
log.info('updating py-tools')
wrap_shell(f'rsync -a {submodule_src_dir}/lichking/ {home}/tools/lichking')
wrap_shell(f'rsync -a {submodule_src_dir}/volumen/ {home}/tools/volumen')
log.info('recreating links & autocompletion for tools')
wrap_shell(f'sudo rm -f /usr/bin/lichking')
wrap_shell(f'sudo rm -f /usr/bin/lich')
wrap_shell(f'sudo rm -f /usr/bin/king')
wrap_shell(f'sudo rm -f /usr/bin/volumen')
wrap_shell(f'sudo rm -f /usr/bin/watchmake')
wrap_shell(f'sudo rm -f /etc/bash_completion.d/cliglue_*')
wrap_shell(f'sudo rm -f /etc/bash_completion.d/nuclear_*')
wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash lichking')
wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash lich')
wrap_shell(f'{home}/tools/lichking/lichking.py --install-bash king')
wrap_shell(f'{home}/tools/watchmake/watchmake.py --install-bash watchmake')
wrap_shell(f'{home}/tools/volumen/volumen.py --install-bash volumen')
wrap_shell(f'diffs --install-autocomplete')
wrap_shell(f'copymon --install-autocomplete')
wrap_shell(f'regex-rename --install-autocomplete')
wrap_shell(f'trimmer --install-autocomplete')
log.info('updating live dev repos')
wrap_shell(f'rm -rf {home}/dev-live')
wrap_shell(f'mkdir -p {home}/dev-live')
for repo_name, url in repo_remotes.items():
log.info(f'initializing live git repo {repo_name}')
repo_path = f'{home}/dev-live/{repo_name}'
wrap_shell(f'mkdir -p {repo_path}')
set_workdir(repo_path)
wrap_shell(f'git init')
wrap_shell(f'git remote add origin "{url}"')
set_workdir(watchmaker_repo)
log.info('clearing gradle cache')
wrap_shell(f'rm -rf {home}/.gradle/*')
log.info('clearing apt cache')
wrap_shell(f'sudo apt clean')
version_file = '/home/user/.osversion'
version_line = read_file(version_file).splitlines()[0]
version_matcher = re.compile(r'^v([0-9]+)\.([0-9]+)$')
match = version_matcher.match(version_line)
assert match
major_version = int(match.group(1))
minor_version = int(match.group(2)) + 1
new_version = f'v{major_version}.{minor_version}'
log.info(f'updating new OS version {new_version}')
save_file(version_file, new_version)
def normalize_song(mp3_file: str,
no_trim: bool,
no_fade: bool,
no_normalize: bool,
user_trim_start: Optional[float] = None,
user_trim_end: Optional[float] = None,
user_gain: Optional[float] = None):
with wrap_context('normalizing mp3', mp3_file=mp3_file):
log.info('loading song...', mp3_file=mp3_file)
song = AudioSegment.from_mp3(mp3_file)
if not no_normalize:
if user_gain is not None:
gain = user_gain
else:
volume = calculate_volume(song)
log.info('normalizing volume level...',
volume=f'{volume:.2f}dB',
dBFS=f'{song.dBFS:.2f}dB')
gain = -volume
song = song.apply_gain(gain)
log.info('volume normalized', gain=f'{gain:.2f}dB')
if not no_trim:
log.info('trimming silence...')
start_trim = user_trim_start * 1000 if user_trim_start is not None else detect_leading_silence(
song)
end_trim = user_trim_end * 1000 if user_trim_end is not None else detect_leading_silence(
song.reverse(), margin=0)
pre_duration = len(song)
song = song[start_trim:len(song) - end_trim]
post_duration = len(song)
log.info('silence trimmed',
trim_start=duration_to_human(start_trim),
trim_end=duration_to_human(end_trim),
duration_before=duration_to_human(pre_duration),
duration_after=duration_to_human(post_duration))
if not no_fade:
fade_in_duration = 100
fade_out_duration = 1000
log.info('applying fade-in & fade-out...',
fade_in=duration_to_human(fade_in_duration),
fade_out=duration_to_human(fade_out_duration))
song = song.fade_in(fade_in_duration).fade_out(fade_out_duration)
duartion = len(song)
log.info('saving song...',
mp3_file=mp3_file,
duration=duration_to_human(duartion))
song.export(mp3_file, format="mp3")
def list_modules():
log.info(f'Available modules:')
for name, path in optional_modules.items():
print(f'{name} - {path}')
def replicate_os(source_disk: str, target_disk: str):
log.info(f'Cloning {source_disk} to {target_disk}...')
wrap_shell(
f'sudo dd if={source_disk} of={target_disk} bs=64K conv=noerror,sync status=progress'
)
wrap_shell('sync')
def resquash_os(storage_path: str, live_squash: str, exclude_file: str):
today = today_stamp()
squashfs_storage_path = f'{storage_path}/filesystem.squashfs'
tagged_squashfs_path = f'{storage_path}/filesystem-{today}.squashfs'
exclude_file_abs = os.path.abspath(exclude_file)
set_workdir('/')
# ensure mount points are mounted
log.info(f'checking mount points')
assert os.path.exists(
storage_path), f'storage path does not exist: {storage_path}'
assert os.path.exists(
live_squash), f'live squash file does not exist: {live_squash}'
assert os.path.exists(
exclude_file_abs), f'exclude file does not exist: {exclude_file_abs}'
log.info('removing old filesystem copy on storage')
wrap_shell(f'sudo rm -f {squashfs_storage_path}')
wrap_shell('sync')
log.info('squashing filesystem...')
wrap_shell(f'''
sudo mksquashfs \
/bin /boot /dev /etc /home /lib /lib64 /media /mnt /opt /proc /run /root /sbin /srv /sys /tmp /usr /var \
/initrd.img /initrd.img.old /vmlinuz /vmlinuz.old \
{squashfs_storage_path} \
-regex -ef {exclude_file_abs} \
-comp gzip -b 512k \
-keep-as-directory
''')
log.info(f'creating tagged copy: {tagged_squashfs_path}...')
wrap_shell(f'sudo cp {squashfs_storage_path} {tagged_squashfs_path}')
wrap_shell('sync')
log.info(f'cheking current squashfs size')
live_squash_mib = os.path.getsize(live_squash) / 1024**2
log.info(f'[!] Putting Live system at risk')
log.info(f'[!] removing current Live squashfs: {live_squash}')
wrap_shell(f'sudo rm -f {live_squash}')
log.info('[!] replacing with newest squashfs')
wrap_shell(
f'sudo rsync -ah --progress --no-perms --no-owner --no-group {squashfs_storage_path} {live_squash}'
)
wrap_shell('sync')
log.info(f'[!] Live system is functional again')
log.info(f'calculating checksum {squashfs_storage_path}')
cksum1 = checksum_file(squashfs_storage_path)
log.info(f'calculating checksum {live_squash}')
cksum2 = checksum_file(live_squash)
assert cksum1 == cksum2
log.info(f'checksums are valid')
tagged_squashfs_mib = os.path.getsize(tagged_squashfs_path) / 1024**2
squash_size_diff = tagged_squashfs_mib - live_squash_mib
log.info(
f'Success. '
f'Resquashed {live_squash}. '
f'Filesystem snaposhot dumped to {tagged_squashfs_path}',
size=f'{tagged_squashfs_mib}MiB',
size_diff=f'{squash_size_diff}MiB')
def flash_disk(disk: str, persistence: bool, boot_storage_surplus: int, modules: List[str], skip_fs: bool):
set_workdir(os.path.join(script_real_dir(), '..'))
log.info(f'checking required files existence')
assert os.path.exists('squash/filesystem.squashfs')
assert os.path.exists('content/boot-files')
assert os.path.exists('content/grub')
assert os.path.exists('modules/init')
# TODO unmount disk partitions
log.warn(f'writing to disk {disk}')
wrap_shell(f'df {disk}')
log.info('creating MBR')
wrap_shell(f'''sudo wipefs {disk}''')
wrap_shell(f'''sudo dd if=/dev/zero of={disk} seek=1 count=2047''')
wrap_shell(f'''
sudo parted --script {disk} \\
mklabel msdos
''')
log.info('calculating partitions size')
# depend on filesystem.squash size, expand by some surplus (for storage)
squashfs_size = os.path.getsize('squash/filesystem.squashfs')
boot_part_min_size = squashfs_size + dir_size('content/boot-files') + dir_size('content/grub')
boot_part_end_mib = boot_part_min_size / 1024 ** 2 + boot_storage_surplus
efi_part_end_mib = boot_part_end_mib + efi_part_size
persistence_part_end_mib = efi_part_end_mib + persistence_part_size
log.info(f'boot partition size: {boot_part_end_mib}MiB ({boot_storage_surplus}MiB surplus)')
log.info('creating partitions space')
if persistence:
wrap_shell(f'''
sudo parted --script {disk} \\
mkpart primary fat32 1MiB {boot_part_end_mib}MiB \\
set 1 lba on \\
set 1 boot on \\
mkpart primary fat32 {boot_part_end_mib}MiB {efi_part_end_mib}MiB \\
set 2 esp on \\
mkpart primary ext4 {efi_part_end_mib}MiB {persistence_part_end_mib}MiB \\
mkpart primary ext4 {persistence_part_end_mib}MiB 100%
''')
else:
wrap_shell(f'''
sudo parted --script {disk} \\
mkpart primary fat32 1MiB {boot_part_end_mib}MiB \\
set 1 lba on \\
set 1 boot on \\
mkpart primary fat32 {boot_part_end_mib}MiB {efi_part_end_mib}MiB \\
set 2 esp on \\
mkpart primary ext4 {efi_part_end_mib}MiB 100%
''')
wrap_shell('sync')
log.info('making boot partition filesystem')
wrap_shell(f'''sudo mkfs.fat -F32 {disk}1''')
log.info('making EFI partition filesystem')
wrap_shell(f'''sudo mkfs.fat -F32 {disk}2''')
if persistence:
log.info('making persistence partition filesystem')
wrap_shell(f'''sudo mkfs.ext4 -F {disk}3''')
log.info('making watchmodules partition filesystem')
wrap_shell(f'''sudo mkfs.ext4 -F {disk}4''')
else:
log.info('making watchmodules partition filesystem')
wrap_shell(f'''sudo mkfs.ext4 -F {disk}3''')
wrap_shell('sync')
log.info('setting partition names')
if persistence:
wrap_shell(f'''
sudo mlabel -i {disk}1 ::boot
sudo mlabel -i {disk}2 ::EFI
sudo e2label {disk}3 persistence
sudo e2label {disk}4 watchmodules
''')
else:
wrap_shell(f'''
sudo mlabel -i {disk}1 ::boot
sudo mlabel -i {disk}2 ::EFI
sudo e2label {disk}3 watchmodules
''')
wrap_shell('sync')
log.info('mounting partitions')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker''')
wrap_shell(f'''
sudo mkdir -p /mnt/watchmaker/boot
sudo mount {disk}1 /mnt/watchmaker/boot
''')
wrap_shell(f'''
sudo mkdir -p /mnt/watchmaker/efi
sudo mount {disk}2 /mnt/watchmaker/efi
''')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/watchmodules''')
if persistence:
wrap_shell(f'''
sudo mkdir -p /mnt/watchmaker/persistence
sudo mount {disk}3 /mnt/watchmaker/persistence
''')
wrap_shell(f'''sudo mount {disk}4 /mnt/watchmaker/watchmodules''')
else:
wrap_shell(f'''sudo mount {disk}3 /mnt/watchmaker/watchmodules''')
log.info('installing GRUB EFI bootloaders')
wrap_shell(f'''
sudo grub-install \\
--target=x86_64-efi \\
--efi-directory=/mnt/watchmaker/boot \\
--boot-directory=/mnt/watchmaker/boot/boot \\
--removable --recheck
''')
wrap_shell(f'''
sudo grub-install \\
--target=x86_64-efi \\
--efi-directory=/mnt/watchmaker/efi \\
--boot-directory=/mnt/watchmaker/boot/boot \\
--removable --recheck
''')
log.info('installing GRUB i386-pc bootloader')
wrap_shell(f'''
sudo grub-install \\
--target=i386-pc \\
--boot-directory=/mnt/watchmaker/boot/boot \\
--recheck \\
{disk}
''')
wrap_shell('sync')
log.info('Fixing GRUB EFI by replacing with Debian GRUB')
wrap_shell(f'''
sudo rm /mnt/watchmaker/efi/EFI/BOOT/*
sudo cp -r content/efi/* /mnt/watchmaker/efi/EFI/BOOT/
sudo rm /mnt/watchmaker/boot/EFI/BOOT/*
sudo cp -r content/efi/* /mnt/watchmaker/boot/EFI/BOOT/
sudo cp -r /mnt/watchmaker/efi/EFI/BOOT /mnt/watchmaker/efi/EFI/debian
sudo cp -r /mnt/watchmaker/boot/EFI/BOOT /mnt/watchmaker/boot/EFI/debian
sudo cp -r content/grub/x86_64-efi /mnt/watchmaker/boot/boot/grub/
''')
log.info('making EFI Microsoft workaround')
wrap_shell(f'''
sudo cp -r /mnt/watchmaker/efi/EFI/BOOT /mnt/watchmaker/efi/EFI/Microsoft
sudo cp -r /mnt/watchmaker/boot/EFI/BOOT /mnt/watchmaker/boot/EFI/Microsoft
''')
log.info('GRUB config')
wrap_shell(f'''
sudo cp content/grub/grub.cfg /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/background.png /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/font.pf2 /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/loopback.cfg /mnt/watchmaker/boot/boot/grub/
sudo cp content/grub/GRUB_FINDME /mnt/watchmaker/boot/
''')
log.info('Boot base files')
wrap_shell(f'''
sudo cp -r content/boot-files/[BOOT] /mnt/watchmaker/boot/
sudo cp -r content/boot-files/d-i /mnt/watchmaker/boot/
sudo cp -r content/boot-files/dists /mnt/watchmaker/boot/
sudo cp -r content/boot-files/live /mnt/watchmaker/boot/
sudo cp -r content/boot-files/pool /mnt/watchmaker/boot/
sudo cp -r content/boot-files/.disk /mnt/watchmaker/boot/
''')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/boot/storage''')
log.info('EFI base files')
wrap_shell(f'''
sudo cp -r content/boot-files/[BOOT] /mnt/watchmaker/efi/
sudo cp -r content/boot-files/d-i /mnt/watchmaker/efi/
sudo cp -r content/boot-files/dists /mnt/watchmaker/efi/
sudo cp -r content/boot-files/live /mnt/watchmaker/efi/
sudo cp -r content/boot-files/pool /mnt/watchmaker/efi/
sudo cp -r content/boot-files/.disk /mnt/watchmaker/efi/
''')
if persistence:
log.info('Persistence configuration')
wrap_shell(f'''sudo cp -r content/persistence/persistence.conf /mnt/watchmaker/persistence/''')
log.info('Copying squash filesystem')
if not skip_fs:
wrap_shell(f'''sudo cp squash/filesystem.squashfs /mnt/watchmaker/boot/live/''')
log.info('Adding init module')
wrap_shell(f'''sudo cp -r modules/init /mnt/watchmaker/watchmodules/''')
log.info('Adding dev module')
wrap_shell(f'''sudo mkdir -p /mnt/watchmaker/watchmodules/dev''')
log.info('make watchmodules writable to non-root user')
wrap_shell(f'''sudo chown igrek /mnt/watchmaker/watchmodules -R''')
if modules:
log.info(f'Adding optional modules: {modules}')
target_path = '/mnt/watchmaker/watchmodules'
for module in modules:
install_module.add_module(module, target_path)
log.info('unmounting')
wrap_shell('sync')
wrap_shell(f'''sudo umount /mnt/watchmaker/boot''')
wrap_shell(f'''sudo umount /mnt/watchmaker/efi''')
wrap_shell(f'''sudo umount /mnt/watchmaker/watchmodules''')
if persistence:
wrap_shell(f'''sudo umount /mnt/watchmaker/persistence''')
wrap_shell('sync')
log.info('Success')
def action_monitor_meminfo(sync: bool):
background_thread: Optional[BackgroundExecuteThread] = None
if sync:
background_thread = run_sync_background()
mem_sizes_buffer: List[MemDataPoint] = []
try:
while True:
# rerun sync
if sync and background_thread and not background_thread.is_alive():
log.info('running sync in background...')
background_thread.stop()
background_thread = run_sync_background()
timestamp: float = time.time()
dirty_kb, writeback_kb = get_mem_dirty_writeback()
remaining_kb: int = dirty_kb + writeback_kb
mem_sizes_buffer.append(
MemDataPoint(timestamp, dirty_kb, writeback_kb))
# max buffer size
if len(mem_sizes_buffer) > 10:
mem_sizes_buffer.pop(0)
speed_temp: float = calc_temporary_speed(mem_sizes_buffer)
speed_avg: float = calc_avg_speed(mem_sizes_buffer)
eta_s: float = calc_eta(remaining_kb, speed_avg)
# output values
print_timestamp = CHAR_BOLD + current_time() + CHAR_RESET
print_remaining = CHAR_BOLD + kb_to_human_just(
remaining_kb) + CHAR_RESET
print_temporary_speed = CHAR_BOLD + kb_to_speed_human_just(
speed_temp) + CHAR_RESET
print_avg_speed = CHAR_BOLD + kb_to_speed_human_just(
speed_avg) + CHAR_RESET
print_eta = CHAR_BOLD + seconds_to_human(eta_s).rjust(
8) + CHAR_RESET
# output formatting
if remaining_kb < 100:
print_remaining = CHAR_GREEN + print_remaining
if speed_temp > 0:
print_temporary_speed = CHAR_RED + print_temporary_speed
elif speed_temp == 0:
print_temporary_speed = CHAR_YELLOW + print_temporary_speed
else:
print_temporary_speed = CHAR_GREEN + print_temporary_speed
if speed_avg > 0:
print_avg_speed = CHAR_RED + print_avg_speed
elif speed_avg == 0:
print_avg_speed = CHAR_YELLOW + print_avg_speed
else:
print_avg_speed = CHAR_GREEN + print_avg_speed
if not eta_s:
print_eta = CHAR_YELLOW + print_eta
elif eta_s < 60:
print_eta = CHAR_GREEN + print_eta
elif eta_s > 600:
print_eta = CHAR_RED + print_eta
print(f'[{print_timestamp}] Remaining:{print_remaining}, '
f'Speed:{print_temporary_speed}, '
f'AVG:{print_avg_speed}, '
f'ETA: {print_eta}')
# delay before next loop
inp = input_or_timeout(1)
# sync command
if inp == 's':
if background_thread and background_thread.is_alive():
log.info('already syncing.')
else:
log.info('running sync in background...')
background_thread = run_sync_background()
elif inp == 'q':
return
except KeyboardInterrupt:
# Ctrl + C handling without printing stack trace
print() # new line)
finally:
# cleanup_thread
if background_thread is not None:
background_thread.stop()
