def main():
common.check_filesystem()
pkgmngr = None
for sfile in ('aptitude', 'aptitude-curses', 'synaptic', 'apper'):
for sdir in ('bin', 'sbin', 'usr/bin', 'usr/sbin'):
full_file = misc.join_paths(config.FILESYSTEM_DIR, sdir, sfile)
if os.path.exists(full_file) and os.access(full_file, os.X_OK):
pkgmngr = misc.join_paths(sdir, sfile)
message.sub_debug('Package manager detected', sfile)
if not pkgmngr:
raise (message.exception('No package manager available'))
if not pkgmngr.startswith('aptitude'):
try:
message.sub_info('Allowing local access to X-server')
misc.system_command((misc.whereis('xhost'), '+local:'))
message.sub_info('Executing package manager')
misc.chroot_exec((pkgmngr))
finally:
message.sub_info('Blocking local access to X-server')
misc.system_command((misc.whereis('xhost'), '-local:'))
else:
message.sub_info('Executing package manager')
misc.chroot_exec((pkgmngr))
def main():
common.check_filesystem()
message.sub_info('Detecting available sessions')
xsession = None
for sfile in misc.list_files(misc.join_paths(config.FILESYSTEM_DIR, \
'usr/share/xsessions')):
if sfile.endswith('.desktop'):
xsession = common.get_value(sfile, 'Exec=')
message.sub_debug('Session detected', xsession)
if not xsession:
raise(message.exception('No session available'))
# FIXME: race condition between session and Xephyr - if session
# starts before Xephyr it fails saying it does not find the DISPLAY
message.sub_info('Starting Xephyr')
x = subprocess.Popen((misc.whereis('Xephyr'), '-ac', '-screen', \
config.RESOLUTION, '-br', ':13'))
x.poll()
if x.returncode is not None and x.returncode > 0:
raise(message.exception('Failed to start Xephyr', x.returncode))
try:
message.sub_info('Allowing local access to X-server')
misc.system_command((misc.whereis('xhost'), '+local:13'))
message.sub_info('Starting nested X session', xsession)
misc.chroot_exec((xsession), xnest=True)
message.sub_info('Blocking local access to X-server')
misc.system_command((misc.whereis('xhost'), '-local:13'))
finally:
message.sub_info('Terminating Xephyr')
x.terminate()
def main():
common.check_filesystem()
pkgmngr = None
for sfile in ('aptitude', 'aptitude-curses', 'synaptic', 'apper'):
for sdir in ('bin', 'sbin', 'usr/bin', 'usr/sbin'):
full_file = misc.join_paths(config.FILESYSTEM_DIR, sdir, sfile)
if os.path.exists(full_file) and os.access(full_file, os.X_OK):
pkgmngr = misc.join_paths(sdir, sfile)
message.sub_debug('Package manager detected', sfile)
if not pkgmngr:
raise(message.exception('No package manager available'))
if not pkgmngr.startswith('aptitude'):
try:
message.sub_info('Allowing local access to X-server')
misc.system_command((misc.whereis('xhost'), '+local:'))
message.sub_info('Executing package manager')
misc.chroot_exec((pkgmngr))
finally:
message.sub_info('Blocking local access to X-server')
misc.system_command((misc.whereis('xhost'), '-local:'))
else:
message.sub_info('Executing package manager')
misc.chroot_exec((pkgmngr))
def main():
common.check_filesystem()
message.sub_info('Detecting available sessions')
xsession = None
for sfile in misc.list_files(misc.join_paths(config.FILESYSTEM_DIR, \
'usr/share/xsessions')):
if sfile.endswith('.desktop'):
xsession = common.get_value(sfile, 'Exec=')
message.sub_debug('Session detected', xsession)
if not xsession:
raise (message.exception('No session avaialable'))
# FIXME: race condition between session and Xephyr - if session
# starts before Xephyr it fails saying it does not find the DISPLAY
message.sub_info('Starting Xephyr')
x = subprocess.Popen((misc.whereis('Xephyr'), '-ac', '-screen', \
config.RESOLUTION, '-br', ':13'))
x.poll()
if x.returncode > 0:
raise (message.exception('Failed to start Xephyr', x.returncode))
try:
message.sub_info('Allowing local access to X-server')
misc.system_command((misc.whereis('xhost'), '+local:13'))
message.sub_info('Starting nested X session', xsession)
misc.chroot_exec((xsession), xnest=True)
message.sub_info('Blocking local access to X-server')
misc.system_command((misc.whereis('xhost'), '-local:13'))
finally:
message.sub_info('Terminating Xephyr')
x.terminate()
def copy_file(source, destination):
if FILE_DEBUG:
message.sub_debug('File {} copied to'.format(source), destination)
base = os.path.dirname(destination)
if not os.path.isdir(base):
os.makedirs(base)
shutil.copyfile(source, destination)
def append_file(sfile, content):
if FILE_DEBUG: message.sub_debug('Appending data to', sfile)
dirname = os.path.dirname(sfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
afile = open(sfile, 'a')
afile.write(content)
afile.close()
def append_file(sfile, content):
if FILE_DEBUG: message.sub_debug('Appending data to', sfile)
dirname = os.path.dirname(sfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
afile = open(sfile, 'a')
afile.write(content)
afile.close()
def list_files(directory):
if FILE_DEBUG: message.sub_debug('Listing files in', directory)
slist = []
if not os.path.exists(directory):
return slist
for root, subdirs, files in os.walk(directory):
for sfile in files:
slist.append(os.path.join(root, sfile))
return slist
def list_files(directory):
if FILE_DEBUG: message.sub_debug('Listing files in', directory)
slist = []
if not os.path.exists(directory):
return slist
for root, subdirs, files in os.walk(directory):
for sfile in files:
slist.append(os.path.join(root, sfile))
return slist
def detect_boot():
global initrd, vmlinuz
initrd = None
vmlinuz = None
for sfile in sorted(misc.list_files(misc.join_paths(config.FILESYSTEM_DIR, 'boot'))):
if 'initrd.img' in sfile:
initrd = sfile
message.sub_debug('Initrd detected', sfile)
elif 'vmlinuz' in sfile:
vmlinuz = sfile
message.sub_debug('Vmlinuz detected', sfile)
def check_filesystem():
message.sub_info('Checking')
corrupted = False
for sdir in ('bin', 'sbin', 'usr/bin', 'usr/sbin', 'etc', 'lib', 'usr/lib'):
full_dir = misc.join_paths(config.FILESYSTEM_DIR, sdir)
if not os.path.isdir(full_dir):
message.sub_debug('Non-existing path', full_dir)
corrupted = True
break
if corrupted:
raise(message.exception('Filesystem is missing or corrupted'))
def check_filesystem():
message.sub_info('Checking')
corrupted = False
for sdir in ('bin', 'sbin', 'usr/bin', 'usr/sbin', 'etc', 'lib',
'usr/lib'):
full_dir = misc.join_paths(config.FILESYSTEM_DIR, sdir)
if not os.path.isdir(full_dir):
message.sub_debug('Non-existing path', full_dir)
corrupted = True
break
if corrupted:
raise (message.exception('Filesystem is missing or corrupted'))
def write_file(sfile, content):
if FILE_DEBUG: message.sub_debug('Writing data to', sfile)
dirname = os.path.dirname(sfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
original = None
if os.path.isfile(sfile):
original = read_file(sfile)
wfile = open(sfile, 'w')
try:
wfile.write(content)
except:
if original:
wfile.write(original)
raise
finally:
wfile.close()
def main():
common.check_filesystem()
if not os.path.isfile(config.HOOK):
raise(message.exception('HOOK does not exists', config.HOOK))
message.sub_info('Copying HOOK file')
hook_file = misc.join_paths(config.FILESYSTEM_DIR, 'hook')
if os.path.isfile(hook_file):
message.sub_debug('Removing', hook_file)
os.unlink(hook_file)
misc.copy_file(config.HOOK, hook_file)
message.sub_info('Making HOOK executable')
os.chmod(hook_file, stat.S_IEXEC)
message.sub_info('Running HOOK')
misc.chroot_exec(('/hook'))
def write_file(sfile, content):
if FILE_DEBUG: message.sub_debug('Writing data to', sfile)
dirname = os.path.dirname(sfile)
if not os.path.isdir(dirname):
os.makedirs(dirname)
original = None
if os.path.isfile(sfile):
original = read_file(sfile)
wfile = open(sfile, 'w')
try:
wfile.write(content)
except:
if original:
wfile.write(original)
raise
finally:
wfile.close()
def main():
common.check_filesystem()
if not os.path.isfile(config.DEB):
raise(message.exception('DEB does not exists', config.DEB))
elif not config.DEB.endswith('.deb'):
raise(message.exception('File is not DEB', config.DEB))
message.sub_info('Copying DEB file')
deb_file = misc.join_paths(config.FILESYSTEM_DIR, 'temp.deb')
if os.path.isfile(deb_file):
message.sub_debug('Removing', deb_file)
os.unlink(deb_file)
misc.copy_file(config.DEB, deb_file)
message.sub_info('Installing DEB')
misc.chroot_exec(('dpkg', '-i', '/temp.deb'))
message.sub_info('Installing dependencies')
misc.chroot_exec(('apt-get', 'install', '-f', '-y'))
def system_command(command, shell=False, cwd=None, env=None):
if CMD_DEBUG: message.sub_debug('Executing system command', command)
if not cwd:
cwd = dir_current()
elif not os.path.isdir(cwd):
cwd = '/'
if not env:
env = os.environ
if isinstance(command, str) and not shell:
command = shlex.split(command)
if CATCH:
pipe = subprocess.Popen(command, stderr=subprocess.PIPE, \
shell=shell, cwd=cwd, env=env)
pipe.wait()
if pipe.returncode != 0:
raise(Exception(pipe.communicate()[1].strip()))
return pipe.returncode
else:
return subprocess.check_call(command, shell=shell, cwd=cwd, env=env)
def main():
common.check_filesystem()
if not os.path.isfile(config.DEB):
raise (message.exception('DEB does not exists', config.DEB))
elif not config.DEB.endswith('.deb'):
raise (message.exception('File is not DEB', config.DEB))
message.sub_info('Copying DEB file')
deb_file = misc.join_paths(config.FILESYSTEM_DIR, 'temp.deb')
if os.path.isfile(deb_file):
message.sub_debug('Removing', deb_file)
os.unlink(deb_file)
misc.copy_file(config.DEB, deb_file)
message.sub_info('Installing DEB')
misc.chroot_exec(('dpkg', '-i', '/temp.deb'))
message.sub_info('Installing dependencies')
misc.chroot_exec(('apt-get', 'install', '-f', '-y'))
def system_command(command, shell=False, cwd=None, env=None):
if CMD_DEBUG: message.sub_debug('Executing system command', command)
if not cwd:
cwd = dir_current()
elif not os.path.isdir(cwd):
cwd = '/'
if not env:
env = os.environ
if isinstance(command, str) and not shell:
command = shlex.split(command)
if CATCH:
pipe = subprocess.Popen(command, stderr=subprocess.PIPE, \
shell=shell, cwd=cwd, env=env)
pipe.wait()
if pipe.returncode != 0:
raise (Exception(pipe.communicate()[1].strip()))
return pipe.returncode
else:
return subprocess.check_call(command, shell=shell, cwd=cwd, env=env)
def clean_work_dirs():
# ensure that nothing is mounted to the working directories before cleanup,
# see https://github.com/clearkimura/Customizer/issues/82
if os.path.exists('/proc/mounts'):
for line in misc.readlines_file('/proc/mounts'):
# spaces are recorded as "\\040", handle them unconditionally
# TODO: test if this actually works with special characters
mpoint = line.split()[1].replace('\\040', ' ')
if mpoint.startswith((config.FILESYSTEM_DIR, config.ISO_DIR)):
message.sub_info('Unmounting', mpoint)
misc.system_command((misc.whereis('umount'), '-f', '-l', mpoint))
else:
message.sub_debug('/proc/mounts does not exists!')
if os.path.isdir(config.FILESYSTEM_DIR):
message.sub_info('Removing', config.FILESYSTEM_DIR)
shutil.rmtree(config.FILESYSTEM_DIR)
if os.path.isdir(config.ISO_DIR):
message.sub_info('Removing', config.ISO_DIR)
shutil.rmtree(config.ISO_DIR)
def main():
common.check_filesystem()
if not os.path.isfile(config.HOOK):
raise (message.exception("HOOK does not exists", config.HOOK))
message.sub_info("Copying HOOK file")
hook_file = misc.join_paths(config.FILESYSTEM_DIR, "hook")
if os.path.isfile(hook_file):
message.sub_debug("Removing", hook_file)
os.unlink(hook_file)
misc.copy_file(config.HOOK, hook_file)
try:
message.sub_info("Making HOOK executable")
os.chmod(hook_file, stat.S_IEXEC)
message.sub_info("Running HOOK")
misc.chroot_exec(("/hook"))
finally:
if os.path.isfile(hook_file):
os.unlink(hook_file)
def clean_work_dirs():
# ensure that nothing is mounted to the working directories before cleanup,
# see https://github.com/clearkimura/Customizer/issues/82
if os.path.exists('/proc/mounts'):
for line in misc.readlines_file('/proc/mounts'):
# spaces are recorded as "\\040", handle them unconditionally
# TODO: test if this actually works with special characters
mpoint = line.split()[1].replace('\\040', ' ')
if mpoint.startswith((config.FILESYSTEM_DIR, config.ISO_DIR)):
message.sub_info('Unmounting', mpoint)
misc.system_command(
(misc.whereis('umount'), '-f', '-l', mpoint))
else:
message.sub_debug('/proc/mounts does not exists!')
if os.path.isdir(config.FILESYSTEM_DIR):
message.sub_info('Removing', config.FILESYSTEM_DIR)
shutil.rmtree(config.FILESYSTEM_DIR)
if os.path.isdir(config.ISO_DIR):
message.sub_info('Removing', config.ISO_DIR)
shutil.rmtree(config.ISO_DIR)
def chroot_exec(command,
prepare=True,
mount=True,
output=False,
xnest=False,
shell=False,
cwd=None):
if CHROOT_DEBUG:
message.sub_debug('Trying to set up chroot command', command)
out = None
resolv = '{}/etc/resolv.conf'.format(config.FILESYSTEM_DIR)
hosts = '{}/etc/hosts'.format(config.FILESYSTEM_DIR)
inhibit = '{}/usr/sbin/policy-rc.d'.format(config.FILESYSTEM_DIR)
mount = whereis('mount')
umount = whereis('umount')
chroot = whereis('chroot')
if isinstance(command, str):
chroot_command = '{} {} {}'.format(chroot, config.FILESYSTEM_DIR,
command)
else:
chroot_command = [chroot, config.FILESYSTEM_DIR]
chroot_command.extend(command)
try:
if prepare:
if CHROOT_DEBUG:
message.sub_debug(
'Preparing chroot environment for networking')
if os.path.isfile(
'/etc/resolv.conf') and not os.path.islink(resolv):
copy_file('/etc/resolv.conf', resolv)
elif os.path.islink(resolv):
# usually /run/resolvconf/resolv.conf
resolv = os.path.realpath(resolv)
rdir = os.path.dirname(resolv)
if not os.path.isdir(rdir):
os.makedirs(rdir)
copy_file('/etc/resolv.conf', resolv)
if os.path.isfile('/etc/hosts'):
if os.path.isfile(hosts):
copy_file(hosts, '{}.backup'.format(hosts))
copy_file('/etc/hosts', hosts)
if mount:
if CHROOT_DEBUG: message.sub_debug('Mounting paths inside chroot')
pseudofs = [
'/proc', '/dev', '/dev/pts', '/dev/shm', '/sys', '/tmp',
'/var/lib/dbus'
]
if os.path.islink(config.FILESYSTEM_DIR + '/var/run'):
pseudofs.append('/run/dbus')
else:
pseudofs.append('/var/run/dbus')
for s in pseudofs:
if not os.path.exists(s):
continue
sdir = config.FILESYSTEM_DIR + s
if not os.path.ismount(sdir):
if not os.path.isdir(sdir):
os.makedirs(sdir)
if MOUNT_DEBUG:
message.sub_debug('Mounting --bind {}'.format(s), sdir)
system_command((mount, '--bind', s, sdir))
if prepare:
if MOUNT_DEBUG: message.sub_debug('Creating mtab inside chroot')
mtab = '{}/etc/mtab'.format(config.FILESYSTEM_DIR)
if not os.path.isfile(mtab) and not os.path.islink(mtab):
os.symlink('../../proc/mounts', mtab)
if not os.path.isfile(inhibit):
write_file(inhibit, "exit 101")
os.chmod(inhibit, 0o755)
if not config.LOCALES == 'C':
system_command(('locale-gen', config.LOCALES))
if CHROOT_DEBUG: message.sub_debug('Enumerating environment variables')
# all operations on reference to os.environ change the environment!
environment = {}
for item in os.environ:
# skip desktop environment specifiec variables because if a DE is
# run in the chroot it may encounter some issues, e.g. with the
# XDG menus
if item.startswith('KDE_') or item == 'XDG_CURRENT_DESKTOP':
continue
environment[item] = os.environ.get(item)
environment['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin'
environment['HOME'] = '/root'
environment['LC_ALL'] = config.LOCALES
environment['LANGUAGE'] = config.LOCALES
environment['LANG'] = config.LOCALES
environment['USER'] = '******'
environment['CASPER_GENERATE_UUID'] = '1'
if xnest:
environment['HOME'] = '/etc/skel'
environment['XDG_CACHE_HOME'] = '/etc/skel/.cache'
environment['XDG_DATA_HOME'] = '/etc/skel/.local/share'
environment['XDG_CONFIG_HOME'] = '/etc/skel/.config'
environment['DISPLAY'] = ':13'
else:
environment['DEBIAN_FRONTEND'] = 'noninteractive'
# FIXME: is this needed?
# environment['DEBIAN_PRIORITY'] = ''
environment['DEBCONF_NONINTERACTIVE_SEEN'] = 'true'
environment['DEBCONF_NOWARNINGS'] = 'true'
if output:
if CHROOT_DEBUG: message.sub_debug('Entering chroot')
out = get_output(chroot_command)
if CHROOT_DEBUG: message.sub_debug('Exiting chroot')
else:
if CHROOT_DEBUG: message.sub_debug('Entering chroot')
system_command(chroot_command, shell=shell, \
env=environment, cwd=cwd)
if CHROOT_DEBUG: message.sub_debug('Exiting chroot')
finally:
if prepare:
if os.path.isfile('{}.backup'.format(hosts)):
copy_file('{}.backup'.format(hosts), hosts)
os.unlink('{}.backup'.format(hosts))
if os.path.isfile(inhibit):
os.unlink(inhibit)
if mount:
for s in reversed(pseudofs):
sdir = config.FILESYSTEM_DIR + s
if os.path.ismount(sdir):
if MOUNT_DEBUG: message.sub_debug('Unmounting -f -l', sdir)
system_command((umount, '-f', '-l', sdir))
time.sleep(0.1) # Wait for lazy unmounts to unlazy...
system_command(('sleep', '1')) # Make sure of it. (and log it)
if output:
return out
def main():
common.check_filesystem()
message.sub_info('Gathering information')
arch = misc.chroot_exec(('dpkg', '--print-architecture'), prepare=False, \
mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_RELEASE=')
message.sub_debug('Architecture', arch)
message.sub_debug('Distribution (DISTRIB_ID)', distrib)
message.sub_debug('Release (DISTRIB_RELEASE)', release)
iso_file = '%s/%s-%s-%s.iso' % (config.WORK_DIR, distrib, arch, release)
if not os.path.exists(iso_file):
raise(message.exception('ISO Image does not exists', iso_file))
message.sub_info('Running QEMU with ISO image', iso_file)
host_arch = os.uname()[4]
if host_arch == 'x86_64':
qemu = misc.whereis('qemu-system-x86_64')
else:
qemu = misc.whereis('qemu-system-i386')
if not qemu:
raise(message.exception('QEMU is not installed'))
qemu_kvm = False
command = [qemu, '-m', config.VRAM, '-cdrom', iso_file]
if misc.search_string('-enable-kvm', misc.get_output((qemu, '-h'))):
qemu_kvm = True
# vmx for intel, svm for amd processors. it will most likely not work in
# XEN environment
host_kvm = False
if os.path.exists('/dev/kvm') and os.path.exists('/proc/cpuinfo') and \
misc.search_file('(?:\\s|^)flags.*(?:\\s)(vme|vmx)(?:\\s|$)', \
'/proc/cpuinfo', escape=False):
host_kvm = True
if qemu_kvm and host_kvm:
command.append('-enable-kvm')
message.sub_debug('Host architecture', host_arch)
message.sub_debug('QEMU KVM', qemu_kvm)
message.sub_debug('Host KVM', host_kvm)
misc.system_command(command)
def main():
common.check_filesystem()
message.sub_info('Gathering information')
arch = misc.chroot_exec(('dpkg', '--print-architecture'), prepare=False, \
mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_RELEASE=')
message.sub_debug('Architecture', arch)
message.sub_debug('Distribution (DISTRIB_ID)', distrib)
message.sub_debug('Release (DISTRIB_RELEASE)', release)
if isinstance(arch, bytes): # For some reason this is of type 'bytes'.
if int(sys.version_info[0]) >= 3: # If we're running under python3
arch = str(arch, 'utf-8')
else: # Otherwise just cast it to a str without the 'utf-8' option.
arch = str(arch)
iso_file = '%s/%s-%s-%s.iso' % (config.WORK_DIR, distrib, arch, release)
if not os.path.exists(iso_file):
raise(message.exception('ISO image does not exist', iso_file))
message.sub_info('Running QEMU with ISO image', iso_file)
host_arch = os.uname()[4]
if host_arch == 'x86_64':
qemu = misc.whereis('qemu-system-x86_64')
else:
qemu = misc.whereis('qemu-system-i386')
if not qemu:
raise(message.exception('QEMU is not installed'))
qemu_kvm = False
command = [qemu, '-m', config.VRAM, '-cdrom', iso_file]
if misc.search_string('-enable-kvm', misc.get_output((qemu, '-h'))):
qemu_kvm = True
# CPU flag: "vmx" for Intel processors, "svm" for AMD processors
# These flags are hidden in Xen environment
# https://wiki.xenproject.org/wiki/Xen_Common_Problems
host_kvm = False
if os.path.exists('/dev/kvm') and os.path.exists('/proc/cpuinfo') and \
misc.search_file('(?:\\s|^)flags.*(?:\\s)(vmx|svm)(?:\\s|$)', \
'/proc/cpuinfo', escape=False):
host_kvm = True
if qemu_kvm and host_kvm:
command.append('-enable-kvm')
message.sub_debug('Host architecture', host_arch)
message.sub_debug('QEMU KVM', qemu_kvm)
message.sub_debug('Host KVM', host_kvm)
misc.system_command(command)
def read_file(sfile):
if FILE_DEBUG: message.sub_debug('Reading entire file', sfile)
rfile = open(sfile, 'r')
content = rfile.read()
rfile.close()
return content
def main():
common.check_filesystem()
# Basic sanity checks of files and paths that absolutely need to exist.
message.sub_info("Doing sanity checks")
lsb_file = misc.join_paths(config.FILESYSTEM_DIR, "etc/lsb-release")
if not os.path.isfile(lsb_file):
raise (message.exception(lsb_file + " does not exist"))
isolinux_dir = misc.join_paths(config.ISO_DIR, "isolinux")
if not os.path.isdir(isolinux_dir):
raise (message.exception(isolinux_dir + " does not exist"))
if misc.search_file("999:999", misc.join_paths(config.FILESYSTEM_DIR, "etc/passwd")):
raise (message.exception("User with UID 999 exists, this mean that automatic login will fail"))
elif misc.search_file("999:999", misc.join_paths(config.FILESYSTEM_DIR, "etc/group")):
raise (message.exception("Group with GID 999 exists, this mean that automatic login will fail"))
casper_dir = misc.join_paths(config.ISO_DIR, "casper")
if not os.path.isdir(casper_dir):
message.sub_debug("Creating", casper_dir)
os.makedirs(casper_dir)
base_file = misc.join_paths(config.ISO_DIR, ".disk/base_installable")
if os.path.isfile(misc.join_paths(config.FILESYSTEM_DIR, "usr/bin/ubiquity")):
if not os.path.isfile(base_file):
message.sub_debug("Creating", base_file)
misc.write_file(base_file, "")
elif os.path.isfile(base_file):
message.sub_debug("Removing", base_file)
os.unlink(base_file)
# Acquire distribution information from the FileSystem
message.sub_info("Gathering information")
arch = misc.chroot_exec(("dpkg", "--print-architecture"), prepare=False, mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + "/etc/lsb-release", "DISTRIB_ID=")
release = common.get_value(config.FILESYSTEM_DIR + "/etc/lsb-release", "DISTRIB_RELEASE=")
# It's really annoying to override the output filename other ways.
# If you actually try to change lsb-release, you end up breaking apt.
livecd_file = misc.join_paths(config.FILESYSTEM_DIR, "etc/livecd-release")
if os.path.isfile(livecd_file):
distrib = common.get_value(config.FILESYSTEM_DIR + "/etc/livecd-release", "DISTRIB_ID=")
release = common.get_value(config.FILESYSTEM_DIR + "/etc/livecd-release", "DISTRIB_RELEASE=")
message.sub_debug("Distribution and Release overriden by /etc/livecd-release")
# Just overwrite the variables if the file actually exists, it's cleaner.
message.sub_debug("Architecture", arch)
message.sub_debug("Distribution (DISTRIB_ID)", distrib)
message.sub_debug("Release (DISTRIB_RELEASE)", release)
# Remove files, by name, that we know we must repopulate if they exist.
message.sub_info("Cleaning up")
cleanup_files = [
"casper/filesystem.squashfs",
"casper/initrd.lz",
"casper/vmlinuz",
"casper/vmlinuz.efi",
"casper/filesystem.manifest",
"casper/filesystem.size",
]
cleanup_files.extend(glob.glob(".disk/casper-uuid-*"))
for sfile in cleanup_files:
full_file = misc.join_paths(config.ISO_DIR, sfile)
if os.path.exists(full_file):
message.sub_debug("Removing", full_file)
os.unlink(full_file)
# Define the checksum files, and the ISO filename.
md5sum_iso_file = misc.join_paths(config.WORK_DIR, "md5sum")
sha1sum_iso_file = misc.join_paths(config.WORK_DIR, "sha1sum")
sha256sum_iso_file = misc.join_paths(config.WORK_DIR, "sha256sum")
iso_file = "%s/%s-%s-%s.iso" % (config.WORK_DIR, distrib, arch, release)
if os.path.exists(iso_file):
message.sub_debug("Removing", iso_file)
os.unlink(iso_file)
if os.path.exists(md5sum_iso_file):
message.sub_debug("Removing", md5sum_iso_file)
os.unlink(md5sum_iso_file)
if os.path.exists(sha1sum_iso_file):
message.sub_debug("Removing", sha1sum_iso_file)
os.unlink(sha1sum_iso_file)
if os.path.exists(sha256sum_iso_file):
message.sub_debug("Removing", sha256sum_iso_file)
os.unlink(sha256sum_iso_file)
# Detect files needed for booting, the kernel, initramfs, xen and anything else.
detect_boot()
if not vmlinuz:
message.sub_info("Re-installing kernel")
misc.chroot_exec(("apt-get", "purge", "--yes", "linux-image*", "-q"))
misc.chroot_exec(("apt-get", "install", "--yes", "linux-signed-generic", "-q"))
misc.chroot_exec(("apt-get", "clean"))
else:
message.sub_info("Updating initramfs")
misc.chroot_exec(("update-initramfs", "-k", "all", "-t", "-u"))
detect_boot()
if not initrd or not vmlinuz:
raise (message.exception("Missing boot file (initrd or vmlinuz)"))
else:
message.sub_info("Copying boot files")
message.sub_debug("Initrd", initrd)
message.sub_debug("Vmlinuz", vmlinuz)
misc.copy_file(initrd, misc.join_paths(config.ISO_DIR, "casper/initrd.lz"))
# FIXME: extend to support grub
efi_boot_entry = False
isolinux_dir = config.ISO_DIR + "/isolinux"
if os.path.isdir(isolinux_dir):
for sfile in os.listdir(isolinux_dir):
if sfile.endswith(".cfg") and misc.search_file("vmlinuz.efi", isolinux_dir + "/" + sfile):
message.sub_debug("Found EFI entry in isolinux conf", sfile)
efi_boot_entry = True
if os.path.isdir(misc.join_paths(config.ISO_DIR, "efi/boot")) or efi_boot_entry:
message.sub_debug("Copying EFI vmlinuz")
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, "casper/vmlinuz.efi"))
os.link(
misc.join_paths(config.ISO_DIR, "casper/vmlinuz.efi"), misc.join_paths(config.ISO_DIR, "casper/vmlinuz")
)
# EFI Kernels are still loadable by grub, modern ISOs lack a bare vmlinuz.
# mkisofs/genisoimage -cache-inodes reuses hard linked inodes.
else:
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, "casper/vmlinuz"))
# We only need to copy the bare kernel if we're not using EFI at all.
# Copy optional boot-enablement packages onto the ISO, if found.
if mt86plus:
message.sub_debug("Memtest86+ kernel", mt86plus)
misc.copy_file(mt86plus, misc.join_paths(config.ISO_DIR, "install/mt86plus"))
if xen_kernel:
message.sub_debug("Xen kernel", xen_kernel)
misc.copy_file(xen_kernel, misc.join_paths(config.ISO_DIR, "casper/" + os.path.basename(xen_kernel)))
if xen_efi:
message.sub_debug("Xen EFI kernel", xen_efi)
misc.copy_file(xen_efi, misc.join_paths(config.ISO_DIR, "casper/" + os.path.basename(xen_efi)))
if ipxe_kernel:
message.sub_debug("iPXE kernel", ipxe_kernel)
misc.copy_file(ipxe_kernel, misc.join_paths(config.ISO_DIR, "casper/" + os.path.basename(ipxe_kernel)))
if ipxe_efi:
message.sub_debug("iPXE EFI kernel", ipxe_efi)
misc.copy_file(ipxe_efi, misc.join_paths(config.ISO_DIR, "casper/" + os.path.basename(ipxe_efi)))
message.sub_info("Extracting casper UUID")
confdir = config.FILESYSTEM_DIR + "/conf"
if os.path.isdir(confdir):
shutil.rmtree(confdir)
os.makedirs(confdir)
try:
misc.chroot_exec(
"zcat " + initrd.replace(config.FILESYSTEM_DIR, "") + " | cpio --quiet -id conf/uuid.conf",
shell=True,
cwd=config.FILESYSTEM_DIR,
)
kernel = re.search("initrd.img-*.*.*-*-(.*)", initrd).group(1)
message.sub_debug("Kernel", kernel)
misc.copy_file(confdir + "/uuid.conf", misc.join_paths(config.ISO_DIR, ".disk/casper-uuid-" + kernel))
finally:
shutil.rmtree(confdir)
# Define some default compression parameters, including a 1MB blocksize for all compressors.
compression_parameters = ("-b", "1048576", "-comp", config.COMPRESSION)
if config.COMPRESSION == "xz": # Append additional compression parameters for xz.
# Using the branch-call-jump filter provides a compression boost with executable code.
# This can save a hundred megabytes easily, on an 800MB ISO. The dictionary size must
# match the block size, and it's advisable to use larger block sizes, like 1MB or 4MB.
compression_parameters += ("-Xbcj", "x86", "-Xdict-size", "100%")
message.sub_info("SquashFS Compression parameters", compression_parameters)
# Create the compressed filesystem
message.sub_info("Creating SquashFS Compressed Filesystem")
make_squash_fs = (
"mksquashfs",
config.FILESYSTEM_DIR,
misc.join_paths(config.ISO_DIR, "casper/filesystem.squashfs"),
"-wildcards",
"-no-recovery",
"-noappend",
"-ef",
os.path.join(sys.prefix, "share/customizer/exclude.list"),
)
misc.system_command(make_squash_fs + compression_parameters)
message.sub_info("Checking SquashFS filesystem size")
sfs_size = os.path.getsize(misc.join_paths(config.ISO_DIR, "casper/filesystem.squashfs"))
message.sub_debug("SquashFS filesystem size", sfs_size)
if sfs_size > 4000000000:
raise (message.exception("The SquashFS filesystem size is greater than 4GB"))
message.sub_info("Creating filesystem.size")
fs_size = 0
for root, subdirs, files in os.walk(config.FILESYSTEM_DIR):
for sfile in files:
sfull = os.path.join(root, sfile)
if os.path.islink(sfull):
continue
# FIXME: respect ignored files from exclude.list
fs_size += os.path.getsize(sfull)
message.sub_debug("Root filesystem size", fs_size)
misc.write_file(misc.join_paths(config.ISO_DIR, "casper/filesystem.size"), str(fs_size))
message.sub_info("Creating filesystem.manifest")
lpackages = misc.chroot_exec(
("dpkg-query", "-W", "--showformat=${Package} ${Version}\\n"), prepare=False, mount=False, output=True
)
message.sub_debug("Packages", lpackages)
misc.write_file(misc.join_paths(config.ISO_DIR, "casper/filesystem.manifest"), lpackages)
# FIXME: do some kung-fu to check if packages are installed
# and remove them from filesystem.manifest-remove if they are not
# Creating a md5sum.txt file fixes lubuntu's integrity check.
md5sums_file = misc.join_paths(config.ISO_DIR, "md5sum.txt")
if os.path.isfile(md5sums_file):
message.sub_info("Creating md5sum.txt")
misc.write_file(md5sums_file, "")
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith("md5sum.txt"):
continue
if sfile.endswith("SHA256SUMS"):
continue
message.sub_debug("MD5 Checksumming", sfile)
checksum = misc.generate_hash_for_file("md5", sfile)
misc.append_file(md5sums_file, checksum + " ." + sfile.replace(config.ISO_DIR, "") + "\n")
# Creating a SHA256SUMS file fixes ubuntu-mini-remix's integrity check.
shasums_file = misc.join_paths(config.ISO_DIR, "SHA256SUMS")
if os.path.isfile(shasums_file):
message.sub_info("Creating SHA256SUMS")
misc.write_file(shasums_file, "")
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith("md5sum.txt"):
continue
if sfile.endswith("SHA256SUMS"):
continue
message.sub_debug("SHA256 Checksumming", sfile)
checksum = misc.generate_hash_for_file("sha256", sfile)
misc.append_file(shasums_file, checksum + " ." + sfile.replace(config.ISO_DIR, "") + "\n")
# Create the ISO filesystem
message.sub_info("Creating ISO")
os.chdir(config.ISO_DIR)
misc.system_command(
(
"xorriso",
"-as",
"mkisofs",
"-r",
"-V",
distrib + "-" + arch + "-" + release,
"-b",
"isolinux/isolinux.bin",
"-c",
"isolinux/boot.cat",
"-J",
"-l",
"-no-emul-boot",
"-boot-load-size",
"4",
"-boot-info-table",
"-o",
iso_file,
"-cache-inodes",
"-input-charset",
"utf-8",
".",
)
)
message.sub_info("Creating ISO checksums")
md5checksum = misc.generate_hash_for_file("md5", iso_file)
message.sub_info("ISO md5 checksum", md5checksum)
misc.append_file(md5sum_iso_file, md5checksum + " ." + iso_file.replace(config.WORK_DIR, "") + "\n")
sha1checksum = misc.generate_hash_for_file("sha1", iso_file)
message.sub_info("ISO sha1 checksum", sha1checksum)
misc.append_file(sha1sum_iso_file, sha1checksum + " ." + iso_file.replace(config.WORK_DIR, "") + "\n")
sha256checksum = misc.generate_hash_for_file("sha256", iso_file)
message.sub_info("ISO sha256 checksum", sha256checksum)
misc.append_file(sha256sum_iso_file, sha256checksum + " ." + iso_file.replace(config.WORK_DIR, "") + "\n")
message.sub_info("Successfuly created ISO image", iso_file)
def detect_boot():
global initrd, vmlinuz, mt86plus, xen_kernel, xen_efi, ipxe_kernel, ipxe_efi
initrd = None
vmlinuz = None
mt86plus = None
xen_kernel = None
xen_efi = None
ipxe_kernel = None
ipxe_efi = None
for sfile in sorted(misc.list_files(misc.join_paths(config.FILESYSTEM_DIR, "boot"))):
if "initrd.img" in sfile:
initrd = sfile
message.sub_debug("Initrd detected", sfile)
elif "vmlinuz" in sfile:
vmlinuz = sfile
message.sub_debug("Vmlinuz detected", sfile)
elif "memtest86" in sfile:
if "+.bin" in sfile:
# In theory, We want memtest86+.bin, not memtest86+_multiboot.bin
# But in actual practice, they're often the same file. Let's be picky.
mt86plus = sfile
message.sub_debug("Memtest86+ kernel detected", sfile)
elif "xen" in sfile:
if "gz" in sfile:
xen_kernel = sfile
message.sub_debug("Xen Hypervisor kernel detected", sfile)
if "efi" in sfile:
xen_efi = sfile
message.sub_debug("Xen Hypervisor EFI kernel detected", sfile)
elif "ipxe" in sfile:
if "lkrn" in sfile:
ipxe_kernel = sfile
message.sub_debug("iPXE kernel detected", sfile)
if "efi" in sfile:
ipxe_efi = sfile
message.sub_debug("iPXE EFI kernel detected", sfile)
def read_file(sfile):
if FILE_DEBUG: message.sub_debug('Reading entire file', sfile)
rfile = open(sfile, 'r')
content = rfile.read()
rfile.close()
return content
def detect_boot():
global initrd, vmlinuz, mt86plus, xen_kernel, xen_efi, ipxe_kernel, ipxe_efi
initrd = None
vmlinuz = None
mt86plus = None
xen_kernel = None
xen_efi = None
ipxe_kernel = None
ipxe_efi = None
for sfile in sorted(misc.list_files(misc.join_paths(config.FILESYSTEM_DIR, 'boot'))):
if 'initrd.img' in sfile:
initrd = sfile
message.sub_debug('Initrd detected', sfile)
elif 'vmlinuz' in sfile:
vmlinuz = sfile
message.sub_debug('Vmlinuz detected', sfile)
elif 'memtest86' in sfile:
if '+.bin' in sfile:
# In theory, We want memtest86+.bin, not memtest86+_multiboot.bin
# But in actual practice, they're often the same file. Let's be picky.
mt86plus = sfile
message.sub_debug('Memtest86+ kernel detected', sfile)
elif 'xen' in sfile:
if 'gz' in sfile:
xen_kernel = sfile
message.sub_debug('Xen Hypervisor kernel detected', sfile)
if 'efi' in sfile:
xen_efi = sfile
message.sub_debug('Xen Hypervisor EFI kernel detected', sfile)
elif 'ipxe' in sfile:
if 'lkrn' in sfile:
ipxe_kernel = sfile
message.sub_debug('iPXE kernel detected', sfile)
if 'efi' in sfile:
ipxe_efi = sfile
message.sub_debug('iPXE EFI kernel detected', sfile)
def detect_boot():
global initrd, vmlinuz, mt86plus, xen_kernel, xen_efi, ipxe_kernel, ipxe_efi
initrd = None
vmlinuz = None
mt86plus = None
xen_kernel = None
xen_efi = None
ipxe_kernel = None
ipxe_efi = None
for sfile in sorted(misc.list_files(misc.join_paths(config.FILESYSTEM_DIR, 'boot'))):
if 'initrd.img' in sfile:
initrd = sfile
message.sub_debug('Initrd detected', sfile)
elif 'vmlinuz' in sfile:
vmlinuz = sfile
message.sub_debug('Vmlinuz detected', sfile)
elif 'memtest86' in sfile:
if '+.bin' in sfile:
# In theory, We want memtest86+.bin, not memtest86+_multiboot.bin
# But in actual practice, they're often the same file. Let's be picky.
mt86plus = sfile
message.sub_debug('Memtest86+ kernel detected', sfile)
elif 'xen' in sfile:
if 'gz' in sfile:
xen_kernel = sfile
message.sub_debug('Xen Hypervisor kernel detected', sfile)
if 'efi' in sfile:
xen_efi = sfile
message.sub_debug('Xen Hypervisor EFI kernel detected', sfile)
elif 'ipxe' in sfile:
if 'lkrn' in sfile:
ipxe_kernel = sfile
message.sub_debug('iPXE kernel detected', sfile)
if 'efi' in sfile:
ipxe_efi = sfile
message.sub_debug('iPXE EFI kernel detected', sfile)
def main():
global mount_dir
if not os.path.isfile(config.ISO):
raise (message.exception('ISO does not exists', config.ISO))
elif not config.ISO.endswith('.iso'):
raise (message.exception('File is not ISO', config.ISO))
common.clean_work_dirs()
common.create_work_dirs()
message.sub_info('Creating mount directory')
mount_dir = tempfile.mkdtemp(prefix=config.MOUNT_DIR + '/')
message.sub_debug('Mount directory is', mount_dir)
message.sub_info('Mounting ISO', config.ISO)
try:
# load the required kernel modules in cases the are not, if they are
# builtin the command fails so ignoring that. the "-b" argument is to
# ensure blacklisted modules are respected
try:
misc.system_command(
(misc.whereis('modprobe'), '-ba', 'loop', 'iso9660'))
except:
pass
misc.system_command((misc.whereis('mount'), '-t', 'iso9660', '-o', \
'ro,loop', config.ISO, mount_dir))
except:
message.sub_info('Removing', mount_dir)
os.rmdir(mount_dir)
common.clean_work_dirs()
raise
message.sub_info('Checking ISO')
for spath in (mount_dir + '/casper/filesystem.squashfs', \
mount_dir + '/casper/filesystem.manifest', \
mount_dir + '/casper/filesystem.manifest-remove', \
mount_dir + '/.disk', mount_dir + '/isolinux', ):
if not os.path.exists(spath):
message.sub_debug('Non-existing path', spath)
common.clean_work_dirs()
unmount_iso()
raise (message.exception('Invalid ISO', config.ISO))
message.sub_info('Unsquashing filesystem')
try:
misc.system_command((misc.whereis('unsquashfs'), '-f', '-d', \
config.FILESYSTEM_DIR, mount_dir + '/casper/filesystem.squashfs'))
except:
unmount_iso()
raise
message.sub_info('Checking architecture')
fs_arch = misc.chroot_exec(('dpkg', '--print-architecture'), \
prepare=False, mount=False, output=True)
host_arch = os.uname()[4]
message.sub_debug('Filesystem architecture', fs_arch)
message.sub_debug('Host architecture', host_arch)
if fs_arch == 'amd64' and not host_arch == 'x86_64':
message.sub_debug('The ISO architecture is amd64 and yours is not')
common.clean_work_dirs()
unmount_iso()
raise (message.exception(
'The ISO architecture is amd64 and yours is not'))
message.sub_info('Copying ISO files')
for sfile in misc.list_files(mount_dir):
if sfile.endswith('casper/filesystem.squashfs'):
continue
else:
message.sub_debug('Copying', sfile)
misc.copy_file(sfile, sfile.replace(mount_dir, config.ISO_DIR))
unmount_iso()
def chroot_exec(command, prepare=True, mount=True, output=False, xnest=False, shell=False, cwd=None):
if CHROOT_DEBUG: message.sub_debug('Trying to set up chroot command', command)
out = None
resolv = '{}/etc/resolv.conf'.format(config.FILESYSTEM_DIR)
hosts = '{}/etc/hosts'.format(config.FILESYSTEM_DIR)
inhibit = '{}/usr/sbin/policy-rc.d'.format(config.FILESYSTEM_DIR)
mount = whereis('mount')
umount = whereis('umount')
chroot = whereis('chroot')
if isinstance(command, str):
chroot_command = '{} {} {}'.format(chroot, config.FILESYSTEM_DIR, command)
else:
chroot_command = [chroot, config.FILESYSTEM_DIR]
chroot_command.extend(command)
try:
if prepare:
if CHROOT_DEBUG: message.sub_debug('Preparing chroot environment for networking')
if os.path.isfile('/etc/resolv.conf') and not os.path.islink(resolv):
copy_file('/etc/resolv.conf', resolv)
elif os.path.islink(resolv):
# usually /run/resolvconf/resolv.conf
resolv = os.path.realpath(resolv)
rdir = os.path.dirname(resolv)
if not os.path.isdir(rdir):
os.makedirs(rdir)
copy_file('/etc/resolv.conf', resolv)
if os.path.isfile('/etc/hosts'):
if os.path.isfile(hosts):
copy_file(hosts, '{}.backup'.format(hosts))
copy_file('/etc/hosts', hosts)
if mount:
if CHROOT_DEBUG: message.sub_debug('Mounting paths inside chroot')
pseudofs = ['/proc', '/dev', '/dev/pts', '/dev/shm', '/sys', '/tmp', '/var/lib/dbus']
if os.path.islink(config.FILESYSTEM_DIR + '/var/run'):
pseudofs.append('/run/dbus')
else:
pseudofs.append('/var/run/dbus')
for s in pseudofs:
if not os.path.exists(s):
continue
sdir = config.FILESYSTEM_DIR + s
if not os.path.ismount(sdir):
if not os.path.isdir(sdir):
os.makedirs(sdir)
if MOUNT_DEBUG: message.sub_debug('Mounting --bind {}'.format(s), sdir)
system_command((mount, '--bind', s, sdir))
if prepare:
if MOUNT_DEBUG: message.sub_debug('Creating mtab inside chroot')
mtab = '{}/etc/mtab'.format(config.FILESYSTEM_DIR)
if not os.path.isfile(mtab) and not os.path.islink(mtab):
os.symlink('../../proc/mounts', mtab)
if not os.path.isfile(inhibit):
write_file(inhibit, "exit 101")
os.chmod(inhibit, 0o755)
if not config.LOCALES == 'C':
system_command(('locale-gen', config.LOCALES))
if CHROOT_DEBUG: message.sub_debug('Enumerating environment variables')
# all operations on reference to os.environ change the environment!
environment = {}
for item in os.environ:
# skip desktop environment specifiec variables because if a DE is
# run in the chroot it may encounter some issues, e.g. with the
# XDG menus
if item.startswith('KDE_') or item == 'XDG_CURRENT_DESKTOP':
continue
environment[item] = os.environ.get(item)
environment['PATH'] = '/usr/sbin:/usr/bin:/sbin:/bin'
environment['HOME'] = '/root'
environment['LC_ALL'] = config.LOCALES
environment['LANGUAGE'] = config.LOCALES
environment['LANG'] = config.LOCALES
environment['USER'] = '******'
environment['CASPER_GENERATE_UUID'] = '1'
if xnest:
environment['HOME'] = '/etc/skel'
environment['XDG_CACHE_HOME'] = '/etc/skel/.cache'
environment['XDG_DATA_HOME'] = '/etc/skel/.local/share'
environment['XDG_CONFIG_HOME'] = '/etc/skel/.config'
environment['DISPLAY'] = ':13'
else:
environment['DEBIAN_FRONTEND'] = 'noninteractive'
# FIXME: is this needed?
# environment['DEBIAN_PRIORITY'] = ''
environment['DEBCONF_NONINTERACTIVE_SEEN'] = 'true'
environment['DEBCONF_NOWARNINGS'] = 'true'
if output:
if CHROOT_DEBUG: message.sub_debug('Entering chroot')
out = get_output(chroot_command)
if CHROOT_DEBUG: message.sub_debug('Exiting chroot')
else:
if CHROOT_DEBUG: message.sub_debug('Entering chroot')
system_command(chroot_command, shell=shell, \
env=environment, cwd=cwd)
if CHROOT_DEBUG: message.sub_debug('Exiting chroot')
finally:
if prepare:
if os.path.isfile('{}.backup'.format(hosts)):
copy_file('{}.backup'.format(hosts), hosts)
os.unlink('{}.backup'.format(hosts))
if os.path.isfile(inhibit):
os.unlink(inhibit)
if mount:
for s in reversed(pseudofs):
sdir = config.FILESYSTEM_DIR + s
if os.path.ismount(sdir):
if MOUNT_DEBUG: message.sub_debug('Unmounting -f -l', sdir)
system_command((umount, '-f', '-l', sdir))
time.sleep(0.1) # Wait for lazy unmounts to unlazy...
system_command(('sleep', '1')) # Make sure of it. (and log it)
if output:
return out
def copy_file(source, destination):
if FILE_DEBUG: message.sub_debug('File {} copied to'.format(source), destination)
base = os.path.dirname(destination)
if not os.path.isdir(base):
os.makedirs(base)
shutil.copyfile(source, destination)
def main():
global mount_dir
if not os.path.isfile(config.ISO):
raise(message.exception('ISO does not exist', config.ISO))
elif not config.ISO.endswith('.iso'):
raise(message.exception('File is not ISO', config.ISO))
common.clean_work_dirs()
common.create_work_dirs()
message.sub_info('Creating mount directory')
mount_dir = tempfile.mkdtemp(prefix=config.MOUNT_DIR + '/')
message.sub_debug('Mount directory is', mount_dir)
message.sub_info('Mounting ISO', config.ISO)
try:
# load the required kernel modules in cases the are not, if they are
# builtin the command fails so ignoring that. the "-b" argument is to
# ensure blacklisted modules are respected
try:
misc.system_command((misc.whereis('modprobe'), '-ba', 'loop', 'iso9660'))
except:
pass
misc.system_command((misc.whereis('mount'), '-t', 'iso9660', '-o', \
'ro,loop', config.ISO, mount_dir))
except:
message.sub_info('Removing', mount_dir)
os.rmdir(mount_dir)
common.clean_work_dirs()
raise
message.sub_info('Checking ISO')
for spath in (mount_dir + '/casper/filesystem.squashfs', \
mount_dir + '/casper/filesystem.manifest', \
mount_dir + '/casper/filesystem.manifest-remove', \
mount_dir + '/.disk', mount_dir + '/isolinux', ):
if not os.path.exists(spath):
message.sub_debug('Non-existing path', spath)
common.clean_work_dirs()
unmount_iso()
raise(message.exception('Invalid ISO', config.ISO))
message.sub_info('Unsquashing filesystem')
try:
misc.system_command((misc.whereis('unsquashfs'), '-f', '-d', \
config.FILESYSTEM_DIR, mount_dir + '/casper/filesystem.squashfs'))
except:
unmount_iso()
raise
message.sub_info('Checking architecture')
fs_arch = misc.chroot_exec(('dpkg', '--print-architecture'), \
prepare=False, mount=False, output=True)
host_arch = os.uname()[4]
message.sub_debug('Filesystem architecture', fs_arch)
message.sub_debug('Host architecture', host_arch)
if fs_arch == 'amd64' and not host_arch == 'x86_64':
message.sub_debug('The ISO architecture is amd64 and yours is not')
common.clean_work_dirs()
unmount_iso()
raise(message.exception('The ISO architecture is amd64 and yours is not'))
message.sub_info('Copying ISO files')
for sfile in misc.list_files(mount_dir):
if sfile.endswith('casper/filesystem.squashfs'):
continue
else:
message.sub_debug('Copying', sfile)
misc.copy_file(sfile, sfile.replace(mount_dir, config.ISO_DIR))
unmount_iso()
def search_file(string, sfile, exact=False, escape=True):
if FILE_DEBUG: message.sub_debug('Searching {} for'.format(sfile), string)
return search_string(string, read_file(sfile), exact=exact, escape=escape)
def readlines_file(sfile):
if FILE_DEBUG: message.sub_debug('Reading lines from', sfile)
rfile = open(sfile, 'r')
content = rfile.readlines()
rfile.close()
return content
def main():
common.check_filesystem()
message.sub_info('Doing sanity checks')
lsb_file = misc.join_paths(config.FILESYSTEM_DIR, 'etc/lsb-release')
if not os.path.isfile(lsb_file):
raise(message.exception(lsb_file + ' does not exists'))
isolinux_dir = misc.join_paths(config.ISO_DIR, 'isolinux')
if not os.path.isdir(isolinux_dir):
raise(message.exception(isolinux_dir + ' does not exist'))
if misc.search_file('999:999', misc.join_paths(config.FILESYSTEM_DIR, 'etc/passwd')):
raise(message.exception('User with UID 999 exists, this mean that automatic login will fail'))
elif misc.search_file('999:999', misc.join_paths(config.FILESYSTEM_DIR, 'etc/group')):
raise(message.exception('Group with GID 999 exists, this mean that automatic login will fail'))
casper_dir = misc.join_paths(config.ISO_DIR, 'casper')
if not os.path.isdir(casper_dir):
message.sub_debug('Creating', casper_dir)
os.makedirs(casper_dir)
base_file = misc.join_paths(config.ISO_DIR, '.disk/base_installable')
if os.path.isfile(misc.join_paths(config.FILESYSTEM_DIR, 'usr/bin/ubiquity')):
if not os.path.isfile(base_file):
message.sub_debug('Creating', base_file)
misc.write_file(base_file, '')
elif os.path.isfile(base_file):
message.sub_debug('Removing', base_file)
os.unlink(base_file)
message.sub_info('Gathering information')
arch = misc.chroot_exec(('dpkg', '--print-architecture'), prepare=False, \
mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_RELEASE=')
message.sub_debug('Architecture', arch)
message.sub_debug('Distribution (DISTRIB_ID)', distrib)
message.sub_debug('Release (DISTRIB_RELEASE)', release)
message.sub_info('Cleaning up')
cleanup_files = ['casper/filesystem.squashfs', 'casper/initrd.lz', \
'casper/vmlinuz', 'casper/vmlinuz.efi', 'casper/filesystem.manifest', \
'casper/filesystem.size']
cleanup_files.extend(glob.glob('.disk/casper-uuid-*'))
for sfile in cleanup_files:
full_file = misc.join_paths(config.ISO_DIR, sfile)
if os.path.exists(full_file):
message.sub_debug('Removing', full_file)
os.unlink(full_file)
iso_file = '%s/%s-%s-%s.iso' % (config.WORK_DIR, distrib, arch, release)
if os.path.exists(iso_file):
message.sub_debug('Removing', iso_file)
os.unlink(iso_file)
detect_boot()
if not vmlinuz:
message.sub_info('Re-installing kernel')
misc.chroot_exec(('apt-get', 'purge', '--yes', 'linux-image*', '-q'))
misc.chroot_exec(('apt-get', 'install', '--yes', \
'linux-image-generic', '-q'))
misc.chroot_exec(('apt-get', 'clean'))
else:
message.sub_info('Updating initramfs')
misc.chroot_exec(('update-initramfs', '-k', 'all', '-t', '-u'))
detect_boot()
if not initrd or not vmlinuz:
raise(message.exception('Missing boot file (initrd or vmlinuz)'))
else:
message.sub_info('Copying boot files')
message.sub_debug('Initrd', initrd)
message.sub_debug('Vmlinuz', vmlinuz)
misc.copy_file(initrd, misc.join_paths(config.ISO_DIR, 'casper/initrd.lz'))
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, 'casper/vmlinuz'))
# FIXME: extend to support grub
efi_boot_entry = False
isolinux_dir = config.ISO_DIR + '/isolinux'
if os.path.isdir(isolinux_dir):
for sfile in os.listdir(isolinux_dir):
if sfile.endswith('.cfg') and misc.search_file('vmlinuz.efi', isolinux_dir + '/' + sfile):
message.sub_debug('Found EFI entry in isolinux conf', sfile)
efi_boot_entry = True
if os.path.isdir(misc.join_paths(config.ISO_DIR, 'efi/boot')) or \
efi_boot_entry:
message.sub_debug('Copying EFI vmlinuz')
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz.efi'))
message.sub_info('Extracting casper UUID')
confdir = config.FILESYSTEM_DIR + '/conf'
if os.path.isdir(confdir):
shutil.rmtree(confdir)
os.makedirs(confdir)
try:
misc.chroot_exec('zcat ' + initrd.lstrip(config.FILESYSTEM_DIR) + ' | ' + ' cpio --quiet -id conf/uuid.conf', shell=True)
kernel = re.search('initrd.img-*.*.*-*-(.*)', initrd).group(1)
message.sub_debug('Kernel', kernel)
misc.copy_file(confdir + '/uuid.conf', misc.join_paths(config.ISO_DIR, \
'.disk/casper-uuid-' + kernel))
finally:
shutil.rmtree(confdir)
message.sub_info('Creating squashed FileSystem')
misc.system_command(('mksquashfs', config.FILESYSTEM_DIR, \
misc.join_paths(config.ISO_DIR, 'casper/filesystem.squashfs'), \
'-wildcards', '-ef', os.path.join(sys.prefix, 'share/customizer/exclude.list'), \
'-comp', config.COMPRESSION))
message.sub_info('Checking SquashFS filesystem size')
sfs_size = os.path.getsize(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.squashfs'))
message.sub_debug('SquashFS filesystem size', sfs_size)
if sfs_size > 4000000000:
raise(message.exception('The SquashFS filesystem size is greater than 4GB'))
message.sub_info('Creating filesystem.size')
fs_size = 0
for root, subdirs, files in os.walk(config.FILESYSTEM_DIR):
for sfile in files:
sfull = os.path.join(root, sfile)
if os.path.islink(sfull):
continue
# FIXME: respect ignored files from exclude.list
fs_size += os.path.getsize(sfull)
message.sub_debug('Root filesystem size', fs_size)
misc.write_file(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.size'), str(fs_size))
message.sub_info('Creating filesystem.manifest')
lpackages = misc.chroot_exec(('dpkg-query', '-W', \
'--showformat=${Package} ${Version}\\n'), prepare=False, mount=False, \
output=True)
message.sub_debug('Packages', lpackages)
misc.write_file(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.manifest'), lpackages)
# FIXME: do some kung-fu to check if packages are installed
# and remove them from filesystem.manifest-remove if they are not
md5sums_file = misc.join_paths(config.ISO_DIR, 'md5sum.txt')
if os.path.isfile(md5sums_file):
message.sub_info('Creating md5sum.txt')
misc.write_file(md5sums_file, '')
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith('md5sum.txt'):
continue
# FIXME: read in chunks
message.sub_debug('Checksuming', sfile)
checksum = hashlib.md5(misc.read_file(sfile)).hexdigest()
misc.append_file(md5sums_file, checksum + ' .' + \
sfile.replace(config.ISO_DIR, '') +'\n')
message.sub_info('Creating ISO')
os.chdir(config.ISO_DIR)
misc.system_command(('xorriso', '-as', 'mkisofs', '-r', '-V', \
distrib + '-' + arch + '-' + release, '-b', 'isolinux/isolinux.bin', \
'-c', 'isolinux/boot.cat', '-J', '-l', '-no-emul-boot', \
'-boot-load-size', '4', '-boot-info-table', '-o', iso_file, \
'-input-charset', 'utf-8', '.'))
message.sub_info('Successfuly created ISO image', iso_file)
def main():
common.check_filesystem()
message.sub_info('Gathering information')
arch = misc.chroot_exec(('dpkg', '--print-architecture'), prepare=False, \
mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_RELEASE=')
message.sub_debug('Architecture', arch)
message.sub_debug('Distribution (DISTRIB_ID)', distrib)
message.sub_debug('Release (DISTRIB_RELEASE)', release)
iso_file = '%s/%s-%s-%s.iso' % (config.WORK_DIR, distrib, arch, release)
if not os.path.exists(iso_file):
raise (message.exception('ISO image does not exist', iso_file))
message.sub_info('Running QEMU with ISO image', iso_file)
host_arch = os.uname()[4]
if host_arch == 'x86_64':
qemu = misc.whereis('qemu-system-x86_64')
else:
qemu = misc.whereis('qemu-system-i386')
if not qemu:
raise (message.exception('QEMU is not installed'))
qemu_kvm = False
command = [qemu, '-m', config.VRAM, '-cdrom', iso_file]
if misc.search_string('-enable-kvm', misc.get_output((qemu, '-h'))):
qemu_kvm = True
# vmx for intel, svm for amd processors. it will most likely not work in
# XEN environment
host_kvm = False
if os.path.exists('/dev/kvm') and os.path.exists('/proc/cpuinfo') and \
misc.search_file('(?:\\s|^)flags.*(?:\\s)(vme|vmx)(?:\\s|$)', \
'/proc/cpuinfo', escape=False):
host_kvm = True
if qemu_kvm and host_kvm:
command.append('-enable-kvm')
message.sub_debug('Host architecture', host_arch)
message.sub_debug('QEMU KVM', qemu_kvm)
message.sub_debug('Host KVM', host_kvm)
misc.system_command(command)
def search_file(string, sfile, exact=False, escape=True):
if FILE_DEBUG: message.sub_debug('Searching {} for'.format(sfile), string)
return search_string(string, read_file(sfile), exact=exact, escape=escape)
def main():
common.check_filesystem()
# Basic sanity checks of files and paths that absolutely need to exist.
message.sub_info('Doing sanity checks')
lsb_file = misc.join_paths(config.FILESYSTEM_DIR, 'etc/lsb-release')
if not os.path.isfile(lsb_file):
raise(message.exception(lsb_file + ' does not exist'))
isolinux_dir = misc.join_paths(config.ISO_DIR, 'isolinux')
if not os.path.isdir(isolinux_dir):
raise(message.exception(isolinux_dir + ' does not exist'))
if misc.search_file('999:999', misc.join_paths(config.FILESYSTEM_DIR, 'etc/passwd')):
raise(message.exception('User with UID 999 exists, this means automatic login will fail'))
elif misc.search_file('999:999', misc.join_paths(config.FILESYSTEM_DIR, 'etc/group')):
raise(message.exception('Group with GID 999 exists, this means automatic login will fail'))
casper_dir = misc.join_paths(config.ISO_DIR, 'casper')
if not os.path.isdir(casper_dir):
message.sub_debug('Creating', casper_dir)
os.makedirs(casper_dir)
base_file = misc.join_paths(config.ISO_DIR, '.disk/base_installable')
if os.path.isfile(misc.join_paths(config.FILESYSTEM_DIR, 'usr/bin/ubiquity')):
if not os.path.isfile(base_file):
message.sub_debug('Creating', base_file)
misc.write_file(base_file, '')
elif os.path.isfile(base_file):
message.sub_debug('Removing', base_file)
os.unlink(base_file)
# Acquire distribution information from the FileSystem
message.sub_info('Gathering information')
arch = misc.chroot_exec(('dpkg', '--print-architecture'), prepare=False, \
mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_RELEASE=')
message.sub_debug('Architecture', arch)
message.sub_debug('Distribution (DISTRIB_ID)', distrib)
message.sub_debug('Release (DISTRIB_RELEASE)', release)
# Remove files, by name, that we know we must repopulate if they exist.
message.sub_info('Cleaning up')
cleanup_files = ['casper/filesystem.squashfs', 'casper/initrd.lz', \
'casper/vmlinuz', 'casper/vmlinuz.efi', 'casper/filesystem.manifest', \
'casper/filesystem.size']
cleanup_files.extend(glob.glob('.disk/casper-uuid-*'))
for sfile in cleanup_files:
full_file = misc.join_paths(config.ISO_DIR, sfile)
if os.path.exists(full_file):
message.sub_debug('Removing', full_file)
os.unlink(full_file)
# Define the checksum files, and the ISO filename.
md5sum_iso_file = misc.join_paths(config.WORK_DIR, 'md5sum')
sha1sum_iso_file = misc.join_paths(config.WORK_DIR, 'sha1sum')
sha256sum_iso_file = misc.join_paths(config.WORK_DIR, 'sha256sum')
iso_file = '%s/%s-%s-%s.iso' % (config.WORK_DIR, distrib, arch, release)
if os.path.exists(iso_file):
message.sub_debug('Removing', iso_file)
os.unlink(iso_file)
if os.path.exists(md5sum_iso_file):
message.sub_debug('Removing', md5sum_iso_file)
os.unlink(md5sum_iso_file)
if os.path.exists(sha1sum_iso_file):
message.sub_debug('Removing', sha1sum_iso_file)
os.unlink(sha1sum_iso_file)
if os.path.exists(sha256sum_iso_file):
message.sub_debug('Removing', sha256sum_iso_file)
os.unlink(sha256sum_iso_file)
# Detect files needed for booting, the kernel, initramfs, xen and anything else.
detect_boot()
if not vmlinuz:
message.sub_info('Re-installing kernel')
misc.chroot_exec(('apt-get', 'purge', '--yes', 'linux-image*', '-q'))
misc.chroot_exec(('apt-get', 'install', '--yes', \
'linux-image-generic', '-q'))
misc.chroot_exec(('apt-get', 'clean'))
else:
message.sub_info('Updating initramfs')
misc.chroot_exec(('update-initramfs', '-k', 'all', '-t', '-u'))
detect_boot()
if not initrd or not vmlinuz:
raise(message.exception('Missing boot file (initrd or vmlinuz)'))
else:
message.sub_info('Copying boot files')
message.sub_debug('Initrd', initrd)
message.sub_debug('Vmlinuz', vmlinuz)
misc.copy_file(initrd, misc.join_paths(config.ISO_DIR, 'casper/initrd.lz'))
# FIXME: extend to support grub
efi_boot_entry = False
isolinux_dir = config.ISO_DIR + '/isolinux'
if os.path.isdir(isolinux_dir):
for sfile in os.listdir(isolinux_dir):
if sfile.endswith('.cfg') and misc.search_file('vmlinuz.efi', isolinux_dir + '/' + sfile):
message.sub_debug('Found EFI entry in isolinux conf', sfile)
efi_boot_entry = True
if os.path.isdir(misc.join_paths(config.ISO_DIR, 'efi/boot')) or \
efi_boot_entry:
message.sub_debug('Copying EFI vmlinuz')
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz.efi'))
os.link(misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz.efi'), misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz'))
# EFI Kernels are still loadable by grub, modern ISOs lack a bare vmlinuz.
# mkisofs/genisoimage -cache-inodes reuses hard linked inodes.
else:
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, 'casper/vmlinuz'))
# We only need to copy the bare kernel if we're not using EFI at all.
# Copy optional boot-enablement packages onto the ISO, if found.
if mt86plus:
message.sub_debug('Memtest86+ kernel', mt86plus)
misc.copy_file(mt86plus, misc.join_paths(config.ISO_DIR, 'install/mt86plus'))
if xen_kernel:
message.sub_debug('Xen kernel', xen_kernel)
misc.copy_file(xen_kernel, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(xen_kernel)))
if xen_efi:
message.sub_debug('Xen EFI kernel', xen_efi)
misc.copy_file(xen_efi, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(xen_efi)))
if ipxe_kernel:
message.sub_debug('iPXE kernel', ipxe_kernel)
misc.copy_file(ipxe_kernel, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(ipxe_kernel)))
if ipxe_efi:
message.sub_debug('iPXE EFI kernel', ipxe_efi)
misc.copy_file(ipxe_efi, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(ipxe_efi)))
message.sub_info('Extracting casper UUID')
confdir = config.FILESYSTEM_DIR + '/conf'
if os.path.isdir(confdir):
shutil.rmtree(confdir)
os.makedirs(confdir)
try:
misc.chroot_exec('zcat ' + initrd.replace(config.FILESYSTEM_DIR, '') + ' | cpio --quiet -id conf/uuid.conf', \
shell=True, cwd=config.FILESYSTEM_DIR)
kernel = re.search('initrd.img-*.*.*-*-(.*)', initrd).group(1)
message.sub_debug('Kernel', kernel)
misc.copy_file(confdir + '/uuid.conf', misc.join_paths(config.ISO_DIR, \
'.disk/casper-uuid-' + kernel))
finally:
shutil.rmtree(confdir)
# Define some default compression parameters, including a 1MB blocksize for all compressors.
compression_parameters = ('-b', '1048576', '-comp', config.COMPRESSION)
if config.COMPRESSION == 'xz': # Append additional compression parameters for xz.
# Using the branch-call-jump filter provides a compression boost with executable code.
# This can save a hundred megabytes easily, on an 800MB ISO. The dictionary size must
# match the block size, and it's advisable to use larger block sizes, like 1MB or 4MB.
compression_parameters += ('-Xbcj', 'x86', '-Xdict-size', '100%')
message.sub_info('SquashFS Compression parameters', compression_parameters)
# Create the compressed filesystem
message.sub_info('Creating SquashFS compressed filesystem')
make_squash_fs = ('mksquashfs', config.FILESYSTEM_DIR, \
misc.join_paths(config.ISO_DIR, 'casper/filesystem.squashfs'), \
'-wildcards', '-no-recovery', '-noappend', \
'-ef', os.path.join(sys.prefix, 'share/customizer/exclude.list'))
misc.system_command(make_squash_fs + compression_parameters)
message.sub_info('Checking SquashFS filesystem size')
sfs_size = os.path.getsize(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.squashfs'))
message.sub_debug('SquashFS filesystem size', sfs_size)
if sfs_size > 4000000000:
raise(message.exception('The SquashFS filesystem size is greater than 4GB'))
message.sub_info('Creating filesystem.size')
fs_size = 0
for root, subdirs, files in os.walk(config.FILESYSTEM_DIR):
for sfile in files:
sfull = os.path.join(root, sfile)
if os.path.islink(sfull):
continue
# FIXME: respect ignored files from exclude.list
fs_size += os.path.getsize(sfull)
message.sub_debug('Root filesystem size', fs_size)
misc.write_file(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.size'), str(fs_size))
message.sub_info('Creating filesystem.manifest')
lpackages = misc.chroot_exec(('dpkg-query', '-W', \
'--showformat=${Package} ${Version}\\n'), prepare=False, mount=False, \
output=True)
message.sub_debug('Packages', lpackages)
misc.write_file(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.manifest'), lpackages)
# FIXME: do some kung-fu to check if packages are installed
# and remove them from filesystem.manifest-remove if they are not
# Creating a md5sum.txt file fixes lubuntu's integrity check.
md5sums_file = misc.join_paths(config.ISO_DIR, 'md5sum.txt')
if os.path.isfile(md5sums_file):
message.sub_info('Creating md5sum.txt')
misc.write_file(md5sums_file, '')
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith('md5sum.txt'):
continue
if sfile.endswith('SHA256SUMS'):
continue
message.sub_debug('MD5 Checksumming', sfile)
checksum = misc.generate_hash_for_file('md5', sfile)
misc.append_file(md5sums_file, checksum + ' .' + \
sfile.replace(config.ISO_DIR, '') +'\n')
# Creating a SHA256SUMS file fixes ubuntu-mini-remix's integrity check.
shasums_file = misc.join_paths(config.ISO_DIR, 'SHA256SUMS')
if os.path.isfile(shasums_file):
message.sub_info('Creating SHA256SUMS')
misc.write_file(shasums_file, '')
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith('md5sum.txt'):
continue
if sfile.endswith('SHA256SUMS'):
continue
message.sub_debug('SHA256 Checksumming', sfile)
checksum = misc.generate_hash_for_file('sha256', sfile)
misc.append_file(shasums_file, checksum + ' .' + \
sfile.replace(config.ISO_DIR, '') +'\n')
# Create the ISO filesystem
message.sub_info('Creating ISO')
os.chdir(config.ISO_DIR)
misc.system_command(('xorriso', '-as', 'mkisofs', '-r', '-V', \
distrib + '-' + arch + '-' + release, '-b', 'isolinux/isolinux.bin', \
'-c', 'isolinux/boot.cat', '-J', '-l', '-no-emul-boot', \
'-boot-load-size', '4', '-boot-info-table', '-o', iso_file, \
'-cache-inodes', '-input-charset', 'utf-8', '.'))
message.sub_info('Creating ISO checksums')
md5checksum = misc.generate_hash_for_file('md5', iso_file)
message.sub_info('ISO md5 checksum', md5checksum)
misc.append_file(md5sum_iso_file, md5checksum + ' .' + \
iso_file.replace(config.WORK_DIR, '') +'\n')
sha1checksum = misc.generate_hash_for_file('sha1', iso_file)
message.sub_info('ISO sha1 checksum', sha1checksum)
misc.append_file(sha1sum_iso_file, sha1checksum + ' .' + \
iso_file.replace(config.WORK_DIR, '') +'\n')
sha256checksum = misc.generate_hash_for_file('sha256', iso_file)
message.sub_info('ISO sha256 checksum', sha256checksum)
misc.append_file(sha256sum_iso_file, sha256checksum + ' .' + \
iso_file.replace(config.WORK_DIR, '') +'\n')
message.sub_info('Successfuly created ISO image', iso_file)
def readlines_file(sfile):
if FILE_DEBUG: message.sub_debug('Reading lines from', sfile)
rfile = open(sfile, 'r')
content = rfile.readlines()
rfile.close()
return content
def main():
common.check_filesystem()
# Basic sanity checks of files and paths that absolutely need to exist.
message.sub_info('Doing sanity checks')
lsb_file = misc.join_paths(config.FILESYSTEM_DIR, 'etc/lsb-release')
if not os.path.isfile(lsb_file):
raise(message.exception(lsb_file + ' does not exist'))
isolinux_dir = misc.join_paths(config.ISO_DIR, 'isolinux')
if not os.path.isdir(isolinux_dir):
raise(message.exception(isolinux_dir + ' does not exist'))
if misc.search_file('999:999', misc.join_paths(config.FILESYSTEM_DIR, 'etc/passwd')):
raise(message.exception('User with UID 999 exists, this means automatic login will fail'))
elif misc.search_file('999:999', misc.join_paths(config.FILESYSTEM_DIR, 'etc/group')):
raise(message.exception('Group with GID 999 exists, this means automatic login will fail'))
casper_dir = misc.join_paths(config.ISO_DIR, 'casper')
if not os.path.isdir(casper_dir):
message.sub_debug('Creating', casper_dir)
os.makedirs(casper_dir)
base_file = misc.join_paths(config.ISO_DIR, '.disk/base_installable')
if os.path.isfile(misc.join_paths(config.FILESYSTEM_DIR, 'usr/bin/ubiquity')):
if not os.path.isfile(base_file):
message.sub_debug('Creating', base_file)
misc.write_file(base_file, '')
elif os.path.isfile(base_file):
message.sub_debug('Removing', base_file)
os.unlink(base_file)
# Acquire distribution information from the FileSystem
message.sub_info('Gathering information')
arch = misc.chroot_exec(('dpkg', '--print-architecture'), prepare=False, \
mount=False, output=True)
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/lsb-release', \
'DISTRIB_RELEASE=')
if isinstance(arch, bytes): # For some reason this is of type 'bytes'.
if int(sys.version_info[0]) >= 3: # If we're running under python3
arch = str(arch, 'utf-8')
else: # Otherwise just cast it to a str without the 'utf-8' option.
arch = str(arch)
# It's really annoying to override the output filename other ways.
# If you actually try to change lsb-release, you end up breaking apt.
livecd_file = misc.join_paths(config.FILESYSTEM_DIR, 'etc/livecd-release')
if os.path.isfile(livecd_file):
distrib = common.get_value(config.FILESYSTEM_DIR + '/etc/livecd-release', \
'DISTRIB_ID=')
release = common.get_value(config.FILESYSTEM_DIR + '/etc/livecd-release', \
'DISTRIB_RELEASE=')
message.sub_debug('Distribution and Release overriden by /etc/livecd-release')
# Just overwrite the variables if the file actually exists, it's cleaner.
message.sub_debug('Architecture', arch)
message.sub_debug('Distribution (DISTRIB_ID)', distrib)
message.sub_debug('Release (DISTRIB_RELEASE)', release)
# Remove files, by name, that we know we must repopulate if they exist.
message.sub_info('Cleaning up')
cleanup_files = ['casper/filesystem.squashfs', 'casper/initrd.lz', \
'casper/vmlinuz', 'casper/vmlinuz.efi', 'casper/filesystem.manifest', \
'casper/filesystem.size']
cleanup_files.extend(glob.glob('.disk/casper-uuid-*'))
for sfile in cleanup_files:
full_file = misc.join_paths(config.ISO_DIR, sfile)
if os.path.exists(full_file):
message.sub_debug('Removing', full_file)
os.unlink(full_file)
# Define the checksum files, and the ISO filename.
md5sum_iso_file = misc.join_paths(config.WORK_DIR, 'md5sum')
sha1sum_iso_file = misc.join_paths(config.WORK_DIR, 'sha1sum')
sha256sum_iso_file = misc.join_paths(config.WORK_DIR, 'sha256sum')
iso_file = '%s/%s-%s-%s.iso' % (config.WORK_DIR, distrib, arch, release)
if os.path.exists(iso_file):
message.sub_debug('Removing', iso_file)
os.unlink(iso_file)
if os.path.exists(md5sum_iso_file):
message.sub_debug('Removing', md5sum_iso_file)
os.unlink(md5sum_iso_file)
if os.path.exists(sha1sum_iso_file):
message.sub_debug('Removing', sha1sum_iso_file)
os.unlink(sha1sum_iso_file)
if os.path.exists(sha256sum_iso_file):
message.sub_debug('Removing', sha256sum_iso_file)
os.unlink(sha256sum_iso_file)
# Detect files needed for booting, the kernel, initramfs, xen and anything else.
detect_boot()
if not vmlinuz:
if config.PURGE_KERNEL:
message.sub_info('Re-installing kernel')
misc.chroot_exec(('apt-get', 'purge', '--yes', 'linux-image*', '-q'))
message.sub_debug("Kernel Selection Debug: {0}, {1}".format(type(config.KERNEL), config.KERNEL))
if config.KERNEL is not "default" or None:
misc.chroot_exec(('apt-get', 'install', '--yes', \
config.KERNEL, '-q'))
else: # use the kernel the user specified in the config.
if arch is not "amd64": # then use the 32bit 'linux-image-generic'
misc.chroot_exec(('apt-get', 'install', '--yes', \
'linux-image-generic', '-q'))
else: # use the amd64 'linux-signed-generic' for uEFI
misc.chroot_exec(('apt-get', 'install', '--yes', \
'linux-signed-generic', '-q'))
misc.chroot_exec(('apt-get', 'clean'))
else:
message.sub_info('Updating initramfs')
misc.chroot_exec(('update-initramfs', '-k', 'all', '-t', '-u'))
detect_boot()
if not initrd or not vmlinuz:
raise(message.exception('Missing boot file (initrd or vmlinuz)'))
else:
message.sub_info('Copying boot files')
message.sub_debug('Initrd', initrd)
message.sub_debug('Vmlinuz', vmlinuz)
misc.copy_file(initrd, misc.join_paths(config.ISO_DIR, 'casper/initrd.lz'))
# FIXME: extend to support grub
efi_boot_entry = False
isolinux_dir = config.ISO_DIR + '/isolinux'
if os.path.isdir(isolinux_dir):
for sfile in os.listdir(isolinux_dir):
if sfile.endswith('.cfg') and misc.search_file('vmlinuz.efi', isolinux_dir + '/' + sfile):
message.sub_debug('Found EFI entry in isolinux conf', sfile)
efi_boot_entry = True
if os.path.isdir(misc.join_paths(config.ISO_DIR, 'efi/boot')) or \
efi_boot_entry:
message.sub_debug('Copying EFI vmlinuz')
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz.efi'))
os.link(misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz.efi'), misc.join_paths(config.ISO_DIR, \
'casper/vmlinuz'))
# EFI Kernels are still loadable by grub, modern ISOs lack a bare vmlinuz.
# mkisofs/genisoimage -cache-inodes reuses hard linked inodes.
else:
misc.copy_file(vmlinuz, misc.join_paths(config.ISO_DIR, 'casper/vmlinuz'))
# We only need to copy the bare kernel if we're not using EFI at all.
# Copy optional boot-enablement packages onto the ISO, if found.
if mt86plus:
message.sub_debug('Memtest86+ kernel', mt86plus)
misc.copy_file(mt86plus, misc.join_paths(config.ISO_DIR, 'install/mt86plus'))
if xen_kernel:
message.sub_debug('Xen kernel', xen_kernel)
misc.copy_file(xen_kernel, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(xen_kernel)))
if xen_efi:
message.sub_debug('Xen EFI kernel', xen_efi)
misc.copy_file(xen_efi, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(xen_efi)))
if ipxe_kernel:
message.sub_debug('iPXE kernel', ipxe_kernel)
misc.copy_file(ipxe_kernel, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(ipxe_kernel)))
if ipxe_efi:
message.sub_debug('iPXE EFI kernel', ipxe_efi)
misc.copy_file(ipxe_efi, \
misc.join_paths(config.ISO_DIR, 'casper/' + os.path.basename(ipxe_efi)))
kernel_type = re.search('initrd.img-*.*.*-*-(.*)', initrd).group(1)
message.sub_debug('Kernel Variant Type', kernel_type)
message.sub_info('Extracting casper UUID')
confdir = config.FILESYSTEM_DIR + '/conf'
if os.path.isdir(confdir):
shutil.rmtree(confdir)
os.makedirs(confdir)
if common.is_gz_file(initrd):
message.sub_debug('Kernel Compression', "gzip without early microcode")
try:
misc.chroot_exec('zcat ' + initrd.replace(config.FILESYSTEM_DIR, '') + ' | cpio --quiet -id conf/uuid.conf', \
shell=True, cwd=config.FILESYSTEM_DIR)
misc.copy_file(confdir + '/uuid.conf', misc.join_paths(config.ISO_DIR, \
'.disk/casper-uuid-' + kernel_type))
finally:
shutil.rmtree(confdir)
else:
message.sub_debug('Kernel Compression', "gzip with early microcode")
try:
misc.chroot_exec('unmkinitramfs ' + initrd.replace(config.FILESYSTEM_DIR, '') + ' conf/', \
shell=True, cwd=config.FILESYSTEM_DIR)
misc.copy_file(confdir + '/main/conf/uuid.conf', misc.join_paths(config.ISO_DIR, \
'.disk/casper-uuid-' + kernel_type))
finally:
shutil.rmtree(confdir)
# Define some default compression parameters, including a 1MB blocksize for all compressors.
compression_parameters = ('-b', '1048576', '-comp', config.COMPRESSION)
if config.COMPRESSION == 'xz': # Append additional compression parameters for xz.
# Using the branch-call-jump filter provides a compression boost with executable code.
# This can save a hundred megabytes easily, on an 800MB ISO. The dictionary size must
# match the block size, and it's advisable to use larger block sizes, like 1MB or 4MB.
compression_parameters += ('-Xbcj', 'x86', '-Xdict-size', '100%')
message.sub_info('SquashFS Compression parameters', compression_parameters)
# Create the compressed filesystem
message.sub_info('Creating SquashFS compressed filesystem')
make_squash_fs = ('mksquashfs', config.FILESYSTEM_DIR, \
misc.join_paths(config.ISO_DIR, 'casper/filesystem.squashfs'), \
'-wildcards', '-no-recovery', '-noappend', \
'-ef', os.path.join(sys.prefix, 'share/customizer/exclude.list'))
misc.system_command(make_squash_fs + compression_parameters)
message.sub_info('Checking SquashFS filesystem size')
sfs_size = os.path.getsize(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.squashfs'))
message.sub_debug('SquashFS filesystem size', sfs_size)
if sfs_size > 4000000000:
raise(message.exception('The SquashFS filesystem size is greater than 4GB'))
message.sub_info('Creating filesystem.size')
fs_size = 0
for root, subdirs, files in os.walk(config.FILESYSTEM_DIR):
for sfile in files:
sfull = os.path.join(root, sfile)
if os.path.islink(sfull):
continue
# FIXME: respect ignored files from exclude.list
fs_size += os.path.getsize(sfull)
message.sub_debug('Root filesystem size', fs_size)
misc.write_file(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.size'), str(fs_size))
message.sub_info('Creating filesystem.manifest')
lpackages = misc.chroot_exec(('dpkg-query', '-W', \
'--showformat=${Package} ${Version}\\n'), prepare=False, mount=False, \
output=True)
if isinstance(lpackages, bytes): # For some reason this is of type 'bytes'.
if int(sys.version_info[0]) >= 3: # If we're running under python3
lpackages = str(lpackages, 'utf-8')
else: # Otherwise just cast it to a str without the 'utf-8' option.
lpackages = str(lpackages)
message.sub_debug('Packages', lpackages)
misc.write_file(misc.join_paths(config.ISO_DIR, \
'casper/filesystem.manifest'), lpackages)
# FIXME: do some kung-fu to check if packages are installed
# and remove them from filesystem.manifest-remove if they are not
# Creating a md5sum.txt file fixes lubuntu's integrity check.
md5sums_file = misc.join_paths(config.ISO_DIR, 'md5sum.txt')
if os.path.isfile(md5sums_file):
message.sub_info('Creating md5sum.txt')
misc.write_file(md5sums_file, '')
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith('md5sum.txt'):
continue
if sfile.endswith('SHA256SUMS'):
continue
message.sub_debug('MD5 Checksumming', sfile)
checksum = misc.generate_hash_for_file('md5', sfile)
misc.append_file(md5sums_file, checksum + ' .' + \
sfile.replace(config.ISO_DIR, '') +'\n')
# Creating a SHA256SUMS file fixes ubuntu-mini-remix's integrity check.
shasums_file = misc.join_paths(config.ISO_DIR, 'SHA256SUMS')
if os.path.isfile(shasums_file):
message.sub_info('Creating SHA256SUMS')
misc.write_file(shasums_file, '')
for sfile in misc.list_files(config.ISO_DIR):
if sfile.endswith('md5sum.txt'):
continue
if sfile.endswith('SHA256SUMS'):
continue
message.sub_debug('SHA256 Checksumming', sfile)
checksum = misc.generate_hash_for_file('sha256', sfile)
misc.append_file(shasums_file, checksum + ' .' + \
sfile.replace(config.ISO_DIR, '') +'\n')
# Create the ISO filesystem
message.sub_info('Creating ISO')
os.chdir(config.ISO_DIR)
misc.system_command(('xorriso', '-as', 'mkisofs', '-r', '-V', \
distrib + '-' + arch + '-' + release, '-isohybrid-mbr',\
'/usr/lib/ISOLINUX/isohdpfx.bin', '-b', 'isolinux/isolinux.bin', \
'-c', 'isolinux/boot.cat', '-J', '-l', '-no-emul-boot', \
'-boot-load-size', '4', '-boot-info-table', '-o', iso_file, \
'-cache-inodes', '-input-charset', 'utf-8', '.'))
message.sub_info('Creating ISO checksums')
md5checksum = misc.generate_hash_for_file('md5', iso_file)
message.sub_info('ISO md5 checksum', md5checksum)
misc.append_file(md5sum_iso_file, md5checksum + ' .' + \
iso_file.replace(config.WORK_DIR, '') +'\n')
sha1checksum = misc.generate_hash_for_file('sha1', iso_file)
message.sub_info('ISO sha1 checksum', sha1checksum)
misc.append_file(sha1sum_iso_file, sha1checksum + ' .' + \
iso_file.replace(config.WORK_DIR, '') +'\n')
sha256checksum = misc.generate_hash_for_file('sha256', iso_file)
message.sub_info('ISO sha256 checksum', sha256checksum)
misc.append_file(sha256sum_iso_file, sha256checksum + ' .' + \
iso_file.replace(config.WORK_DIR, '') +'\n')
message.sub_info('Successfuly created ISO image', iso_file)
