def prepare(self, unfiltered=False): self.language_question = None self.initial_language = None self.db.fset('localechooser/languagelist', 'seen', 'false') with misc.raised_privileges(): osextras.unlink_force('/var/lib/localechooser/preseeded') osextras.unlink_force('/var/lib/localechooser/langlevel') if self.ui.controller.oem_config: try: self.ui.set_oem_id(self.db.get('oem-config/id')) except debconf.DebconfError: pass show = self.db.get('ubiquity/show_alpha_warning') == 'true' self.ui.set_alpha_warning(show) localechooser_script = '/usr/lib/ubiquity/localechooser/localechooser' if ('UBIQUITY_FRONTEND' in os.environ and os.environ['UBIQUITY_FRONTEND'] == 'debconf_ui'): localechooser_script += '-debconf' questions = ['localechooser/languagelist'] environ = { 'PATH': '/usr/lib/ubiquity/localechooser:' + os.environ['PATH'], } if ('UBIQUITY_FRONTEND' in os.environ and os.environ['UBIQUITY_FRONTEND'] == "debconf_ui"): environ['TERM_FRAMEBUFFER'] = '1' else: environ['OVERRIDE_SHOW_ALL_LANGUAGES'] = '1' return localechooser_script, questions, environ
def prepare(self, unfiltered=False): self.language_question = None self.initial_language = None self.db.fset("localechooser/languagelist", "seen", "false") with misc.raised_privileges(): osextras.unlink_force("/var/lib/localechooser/preseeded") osextras.unlink_force("/var/lib/localechooser/langlevel") if self.ui.controller.oem_config: try: self.ui.set_oem_type(self.db.get("oem-config/device-type")) self.ui.set_oem_manufacturer(self.db.get("oem-config/device-manufacturer")) self.ui.set_oem_model(self.db.get("oem-config/device-model")) except debconf.DebconfError: pass show = self.db.get("ubiquity/show_alpha_warning") == "true" self.ui.set_alpha_warning(show) localechooser_script = "/usr/lib/ubiquity/localechooser/localechooser" if "UBIQUITY_FRONTEND" in os.environ and os.environ["UBIQUITY_FRONTEND"] == "debconf_ui": localechooser_script += "-debconf" questions = ["localechooser/languagelist"] environ = {"PATH": "/usr/lib/ubiquity/localechooser:" + os.environ["PATH"]} if "UBIQUITY_FRONTEND" in os.environ and os.environ["UBIQUITY_FRONTEND"] == "debconf_ui": environ["TERM_FRAMEBUFFER"] = "1" else: environ["OVERRIDE_SHOW_ALL_LANGUAGES"] = "1" return (localechooser_script, questions, environ)
def prepare(self, unfiltered=False): self.preseed('console-setup/ask_detect', 'false') # We need to get rid of /etc/default/keyboard, or console-setup will # think it's already configured and behave differently. Try to save # the old file for interest's sake, but it's not a big deal if we # can't. with misc.raised_privileges(): osextras.unlink_force('/etc/default/keyboard.pre-ubiquity') try: os.rename('/etc/default/keyboard', '/etc/default/keyboard.pre-ubiquity') except OSError: osextras.unlink_force('/etc/default/keyboard') # Make sure debconf doesn't do anything with crazy "preseeded" # answers to these questions. If you want to preseed these, use the # *code variants. self.db.fset('keyboard-configuration/layout', 'seen', 'false') self.db.fset('keyboard-configuration/variant', 'seen', 'false') self.db.fset('keyboard-configuration/model', 'seen', 'false') self.db.fset('console-setup/codeset47', 'seen', 'false') # Roughly taken from console-setup's config.proto: di_locale = self.db.get('debian-installer/locale') ret = di_locale.rsplit('.', 1)[0] if not keyboard_names.has_language(ret): self.debug("No keyboard layout translations for locale '%s'" % ret) ret = ret.rsplit('_', 1)[0] if not keyboard_names.has_language(ret): self.debug("No keyboard layout translations for locale '%s'" % ret) # TODO should this be C.UTF-8?! ret = 'C' self._locale = ret self.has_variants = False # Technically we should provide a version as the second argument, # but that isn't currently needed and it would require querying # apt/dpkg for the current version, which would be slow, so we don't # bother for now. command = [ '/usr/lib/ubiquity/console-setup/keyboard-configuration.postinst', 'configure', ] questions = [ '^keyboard-configuration/layout', '^keyboard-configuration/variant', '^keyboard-configuration/model', '^keyboard-configuration/altgr$', '^keyboard-configuration/unsupported_', ] environ = { 'OVERRIDE_ALLOW_PRESEEDING': '1', 'OVERRIDE_USE_DEBCONF_LOCALE': '1', 'LC_ALL': di_locale, 'PATH': '/usr/lib/ubiquity/console-setup:' + os.environ['PATH'], 'DPKG_MAINTSCRIPT_NAME': 'postinst', 'DPKG_MAINTSCRIPT_PACKAGE': 'keyboard-configuration', } return command, questions, environ
def prepare(self, unfiltered=False): self.language_question = None self.initial_language = None self.db.fset('localechooser/languagelist', 'seen', 'false') with misc.raised_privileges(): osextras.unlink_force('/var/lib/localechooser/preseeded') osextras.unlink_force('/var/lib/localechooser/langlevel') if self.ui.controller.oem_config: try: self.ui.set_oem_id(self.db.get('oem-config/id')) except debconf.DebconfError: pass localechooser_script = '/usr/lib/ubiquity/localechooser/localechooser' if ('UBIQUITY_FRONTEND' in os.environ and os.environ['UBIQUITY_FRONTEND'] == 'debconf_ui'): localechooser_script += '-debconf' questions = ['localechooser/languagelist'] environ = { 'PATH': '/usr/lib/ubiquity/localechooser:' + os.environ['PATH'], } if ('UBIQUITY_FRONTEND' in os.environ and os.environ['UBIQUITY_FRONTEND'] == "debconf_ui"): environ['TERM_FRAMEBUFFER'] = '1' else: environ['OVERRIDE_SHOW_ALL_LANGUAGES'] = '1' return localechooser_script, questions, environ
def remove_target(source_root, target_root, relpath, st_source): """Remove a target file if necessary and if we can. On the whole, we can assume that partman-target has arranged to clear out the areas of the filesystem we're installing to. However, in edge cases it's possible that there is still some detritus left over, and we want to steer a reasonable course between cavalierly destroying data and crashing. So, we remove non-directories and empty directories that are in our way, but if a non-empty directory is in our way then we move it aside (adding .bak suffixes until we find something unused) instead. """ targetpath = os.path.join(target_root, relpath) try: st_target = os.lstat(targetpath) except OSError: # The target does not exist. Boring. return if stat.S_ISDIR(st_source.st_mode) and stat.S_ISDIR(st_target.st_mode): # One directory is as good as another, so we don't need to remove an # existing directory just in order to create another one. return if not stat.S_ISDIR(st_target.st_mode): # Installing over a non-directory is easy; just remove it. osextras.unlink_force(targetpath) return try: # Is it an empty directory? That's easy too. os.rmdir(targetpath) return except OSError, e: if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise
def apply_keyboard(): """Set the keyboard layout to the default layout for the language selected. If a user wants a different layout, they can be reasonably expected to change it in System -> Preferences -> Keyboard. """ # Mostly taken from ubi-console-setup. # We need to get rid of /etc/default/keyboard, or keyboard-configuration # will think it's already configured and behave differently. Try to save # the old file for interest's sake, but it's not a big deal if we can't. osextras.unlink_force('/etc/default/keyboard.pre-ubiquity') try: os.rename('/etc/default/keyboard', '/etc/default/keyboard.pre-ubiquity') except OSError: osextras.unlink_force('/etc/default/keyboard') import debconf dccomm = subprocess.Popen( ['debconf-communicate', '-fnoninteractive', 'ubiquity'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True, universal_newlines=True) try: dc = debconf.Debconf(read=dccomm.stdout, write=dccomm.stdin) try: # Make sure debconf doesn't do anything with crazy "preseeded" # answers to these questions. If you want to preseed these, use the # *code variants. dc.fset('keyboard-configuration/layout', 'seen', 'false') dc.fset('keyboard-configuration/variant', 'seen', 'false') dc.fset('keyboard-configuration/model', 'seen', 'false') dc.fset('console-setup/codeset47', 'seen', 'false') except debconf.DebconfError: return finally: dccomm.stdin.close() dccomm.wait() # Accept all the defaults, given the preseeded language. child_env = dict(os.environ) child_env['OVERRIDE_ALLOW_PRESEEDING'] = '1' subprocess.call( ['dpkg-reconfigure', '-fnoninteractive', 'keyboard-configuration'], env=child_env) misc.execute('setupcon', '--save-only') # Reprocess /lib/udev/rules.d/64-xorg-xkb.rules misc.execute('udevadm', 'trigger', '--subsystem-match=input', '--action=change') misc.execute('udevadm', 'settle')
def apply_keyboard(): """Set the keyboard layout to the default layout for the language selected. If a user wants a different layout, they can be reasonably expected to change it in System -> Preferences -> Keyboard. """ # Mostly taken from ubi-console-setup. # We need to get rid of /etc/default/keyboard, or keyboard-configuration # will think it's already configured and behave differently. Try to save # the old file for interest's sake, but it's not a big deal if we can't. osextras.unlink_force('/etc/default/keyboard.pre-ubiquity') try: os.rename('/etc/default/keyboard', '/etc/default/keyboard.pre-ubiquity') except OSError: osextras.unlink_force('/etc/default/keyboard') import debconf dccomm = subprocess.Popen(['debconf-communicate', '-fnoninteractive', 'ubiquity'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True, universal_newlines=True) try: dc = debconf.Debconf(read=dccomm.stdout, write=dccomm.stdin) try: # Make sure debconf doesn't do anything with crazy "preseeded" # answers to these questions. If you want to preseed these, use the # *code variants. dc.fset('keyboard-configuration/layout', 'seen', 'false') dc.fset('keyboard-configuration/variant', 'seen', 'false') dc.fset('keyboard-configuration/model', 'seen', 'false') dc.fset('console-setup/codeset47', 'seen', 'false') except debconf.DebconfError: return finally: dccomm.stdin.close() dccomm.wait() # Accept all the defaults, given the preseeded language. child_env = dict(os.environ) child_env['OVERRIDE_ALLOW_PRESEEDING'] = '1' subprocess.call(['dpkg-reconfigure', '-fnoninteractive', 'keyboard-configuration'], env=child_env) misc.execute('setupcon', '--save-only') # Reprocess /lib/udev/rules.d/64-xorg-xkb.rules misc.execute('udevadm', 'trigger', '--subsystem-match=input', '--action=change') misc.execute('udevadm', 'settle')
def start_debconf(): """debconf_ui needs to run within a debconf frontend.""" if 'DEBIAN_HAS_FRONTEND' in os.environ: # debconf already started, so just clean up the configuration file # if any (debconf has already read it by now). if 'DEBCONF_SYSTEMRC' in os.environ: osextras.unlink_force(os.environ['DEBCONF_SYSTEMRC']) return print("debconf_ui selected; starting debconf frontend", file=sys.stderr) if 'UBIQUITY_AUTOMATIC' in os.environ: # In automatic mode, we don't want debconf to reshow seen questions, # since that defeats the purpose of preseeding. pass elif 'DEBCONF_USE_CDEBCONF' not in os.environ: # This is rather unsatisfactory. Perhaps it would be better to # have a custom debconf program, a bit like dpkg-reconfigure. import tempfile debconfrc_fd, debconfrc = tempfile.mkstemp() os.chmod(debconfrc, 0o644) debconfrc_file = os.fdopen(debconfrc_fd, 'w') orig_debconfrc = open('/etc/debconf.conf') state = 0 for line in orig_debconfrc: if (state == 0 and line.rstrip('\n') and not line.startswith('#')): state = 1 elif state == 1 and not line.rstrip('\n'): print('Reshow: true', file=debconfrc_file) state = 2 print(line, end="", file=debconfrc_file) orig_debconfrc.close() debconfrc_file.close() os.environ['DEBCONF_SYSTEMRC'] = debconfrc else: os.environ['DEBCONF_SHOWOLD'] = 'true' if 'DEBCONF_USE_CDEBCONF' not in os.environ: os.environ['DEBCONF_PACKAGE'] = 'ubiquity' # TODO: need to set owner somehow for the cdebconf case import debconf debconf.runFrontEnd() # re-execs this program
def chroot_cleanup(target, x11=False): """Undo the work done by chroot_setup.""" if target == '/': return if x11 and 'DISPLAY' in os.environ: misc.execute('umount', os.path.join(target, 'tmp/.X11-unix')) try: os.rmdir(os.path.join(target, 'tmp/.X11-unix')) except OSError: pass osextras.unlink_force(os.path.join(target, 'root/.Xauthority')) chrex(target, 'umount', '/sys') chrex(target, 'umount', '/proc') misc.execute('umount', os.path.join(target, 'dev')) initctl = os.path.join(target, 'sbin/initctl') if os.path.exists('%s.REAL' % initctl): os.rename('%s.REAL' % initctl, initctl) start_stop_daemon = os.path.join(target, 'sbin/start-stop-daemon') if os.path.exists('%s.REAL' % start_stop_daemon): os.rename('%s.REAL' % start_stop_daemon, start_stop_daemon) else: osextras.unlink_force(start_stop_daemon) policy_rc_d = os.path.join(target, 'usr/sbin/policy-rc.d') osextras.unlink_force(policy_rc_d)
def chroot_cleanup(target, x11=False): """Undo the work done by chroot_setup.""" if target == '/': return if x11 and 'DISPLAY' in os.environ: misc.execute('umount', os.path.join(target, 'tmp/.X11-unix')) try: os.rmdir(os.path.join(target, 'tmp/.X11-unix')) except OSError: pass osextras.unlink_force(os.path.join(target, 'root/.Xauthority')) chrex(target, 'umount', '/sys') chrex(target, 'umount', '/proc') misc.execute('umount', os.path.join(target, 'run')) misc.execute('umount', os.path.join(target, 'dev')) initctl = os.path.join(target, 'sbin/initctl') if os.path.exists('%s.REAL' % initctl): os.rename('%s.REAL' % initctl, initctl) start_stop_daemon = os.path.join(target, 'sbin/start-stop-daemon') if os.path.exists('%s.REAL' % start_stop_daemon): os.rename('%s.REAL' % start_stop_daemon, start_stop_daemon) else: osextras.unlink_force(start_stop_daemon) policy_rc_d = os.path.join(target, 'usr/sbin/policy-rc.d') osextras.unlink_force(policy_rc_d)
def main(oem_config): force_utf8_locale() usage = '%prog [options] [frontend]' parser = optparse.OptionParser(usage=usage, version=VERSION) parser.set_defaults(debug=('UBIQUITY_DEBUG' in os.environ), debug_pdb=False, cdebconf=False, automatic=False, query=False) parser.add_option('-d', '--debug', dest='debug', action='store_true', help='debug mode (warning: passwords will be logged!)') parser.add_option('--pdb', dest='debug_pdb', action='store_true', help='drop into Python debugger on a crash') parser.add_option('--cdebconf', dest='cdebconf', action='store_true', help='use cdebconf instead of debconf (experimental)') parser.add_option('--automatic', dest='automatic', action='store_true', help='do not ignore the "seen" flag (useful for ' 'unattended installations).') parser.add_option('--only', dest='only', action='store_true', help='tell the application that it is the only desktop ' 'program running so that it can customize its UI ' 'to better suit a minimal environment.') parser.add_option('-q', '--query', dest='query', action='store_true', help='find out which frontend will be used by default') parser.add_option('-g', '--greeter', dest='greeter', action='store_true', help='allow the user to leave the installer and enter ' 'a live desktop (for the initial boot).') parser.add_option('-b', '--no-bootloader', dest='bootloader', default=True, action='store_false', help='Do not install a bootloader.') parser.add_option('--ldtp', dest='ldtp', action='store_true', help='Name widgets in ATK by their GtkBuilder names, ' 'to support LDTP testing.') parser.add_option('--no-webcam', dest='webcam', default=True, action='store_false', help='Disable the webcam page.') (options, args) = parser.parse_args() if options.debug: os.environ['UBIQUITY_DEBUG'] = '1' if options.debug_pdb: os.environ['UBIQUITY_DEBUG_PDB'] = '1' if options.cdebconf: # Note that this needs to be set before DebconfCommunicate is # imported by anything. os.environ['DEBCONF_USE_CDEBCONF'] = '1' prepend_path('/usr/lib/cdebconf') prepend_path('/usr/lib/ubiquity/compat') if options.automatic: os.environ['UBIQUITY_AUTOMATIC'] = '1' if options.greeter: os.environ['UBIQUITY_GREETER'] = '1' if options.only: os.environ['UBIQUITY_ONLY'] = '1' if not options.bootloader: os.environ['UBIQUITY_NO_BOOTLOADER'] = '1' if oem_config: os.environ['UBIQUITY_OEM_USER_CONFIG'] = '1' global TARGET TARGET = '' if options.ldtp: os.environ['UBIQUITY_LDTP'] = '1' if not options.webcam: os.environ['UBIQUITY_NO_WEBCAM'] = '1' acquire_lock() try: os.makedirs('/var/log/installer') except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise syslog.openlog('ubiquity', syslog.LOG_NOWAIT | syslog.LOG_PID) if oem_config: syslog.syslog("Ubiquity %s (oem-config)" % VERSION) else: syslog.syslog("Ubiquity %s" % VERSION) with open('/var/log/installer/version', 'w') as version_file: print('ubiquity %s' % VERSION, file=version_file) if 'UBIQUITY_DEBUG' in os.environ: if 'UBIQUITY_DEBUG_CORE' not in os.environ: os.environ['UBIQUITY_DEBUG_CORE'] = '1' if 'DEBCONF_DEBUG' not in os.environ: os.environ['DEBCONF_DEBUG'] = 'developer' # The frontend should take care of displaying a helpful message if # we are being run without root privileges. if not (options.query and args and args[0] == 'noninteractive'): try: if oem_config: logfile = '/var/log/oem-config.log' else: logfile = '/var/log/installer/debug' log = os.open(logfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) os.dup2(log, 2) os.close(log) sys.stderr = os.fdopen(2, 'a', 1) if oem_config: print("Ubiquity %s (oem-config)" % VERSION, file=sys.stderr) else: print("Ubiquity %s" % VERSION, file=sys.stderr) except (IOError, OSError) as err: if err.errno != errno.EACCES: raise # Default to enabling internal (non-debconf) debugging except for when # using --automatic. if 'UBIQUITY_DEBUG_CORE' not in os.environ: if options.automatic: os.environ['UBIQUITY_DEBUG_CORE'] = '1' # Initialise cdebconf if necessary, to work around # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=451130. if options.cdebconf: for package in ('ubiquity', 'keyboard-configuration', 'console-setup'): subprocess.call([ 'debconf-loadtemplate', package, '/var/lib/dpkg/info/%s.templates' % package ]) # Clean up old state. for name in ('apt-installed', 'apt-install-direct', 'remove-kernels', 'apt-removed', 'encrypted-swap', 'started-installing'): osextras.unlink_force(os.path.join('/var/lib/ubiquity', name)) shutil.rmtree("/var/lib/partman", ignore_errors=True) misc.remove_os_prober_cache() if oem_config and not options.query: disable_autologin() if args: install(args[0], query=options.query) else: install(query=options.query) if oem_config: run_oem_hooks() im_switch.kill_im()
def test_unlink_force_ignores_missing(self): path = os.path.join(self.temp_dir, "missing") self.assertFalse(os.path.exists(path)) osextras.unlink_force(path) self.assertFalse(os.path.exists(path))
if dirty_writeback_centisecs is not None: with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print >>dwc, dirty_writeback_centisecs if dirty_expire_centisecs is not None: with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print >>dec, dirty_expire_centisecs # Try some possible locations for the kernel we used to boot. This # lets us save a couple of megabytes of CD space. bootdir = os.path.join(self.target, 'boot') kernel = self.find_cd_kernel() if kernel: prefix = os.path.basename(kernel).split('-', 1)[0] release = os.uname()[2] target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release)) osextras.unlink_force(target_kernel) install_misc.copy_file(self.db, kernel, target_kernel, md5_check) os.lchown(target_kernel, 0, 0) os.chmod(target_kernel, 0644) st = os.lstat(kernel) try: os.utime(target_kernel, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass os.umask(old_umask) self.db.progress('SET', 100) self.db.progress('STOP')
def release_lock(): global lock osextras.unlink_force(LOCKFILE) if lock is not None: lock.close() lock = None
def remove_os_prober_cache(): osextras.unlink_force('/var/lib/ubiquity/os-prober-cache') shutil.rmtree('/var/lib/ubiquity/linux-boot-prober-cache', ignore_errors=True)
def remove_target(source_root, target_root, relpath, st_source): """Remove a target file if necessary and if we can. On the whole, we can assume that partman-target has arranged to clear out the areas of the filesystem we're installing to. However, in edge cases it's possible that there is still some detritus left over, and we want to steer a reasonable course between cavalierly destroying data and crashing. So, we remove non-directories and empty directories that are in our way, but if a non-empty directory is in our way then we move it aside (adding .bak suffixes until we find something unused) instead. """ targetpath = os.path.join(target_root, relpath) try: st_target = os.lstat(targetpath) except OSError: # The target does not exist. Boring. return if stat.S_ISDIR(st_source.st_mode) and stat.S_ISDIR(st_target.st_mode): # One directory is as good as another, so we don't need to remove an # existing directory just in order to create another one. return if not stat.S_ISDIR(st_target.st_mode): # Installing over a non-directory is easy; just remove it. osextras.unlink_force(targetpath) return try: # Is it an empty directory? That's easy too. os.rmdir(targetpath) return except OSError as e: if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise # If we've got this far, then we must be trying to install a # non-directory over an existing non-empty directory. The slightly # easier case is if it's a symlink, and if the prospective symlink # target hasn't been copied yet or is empty; in that case, we should try # to move the existing directory to the symlink target. if stat.S_ISLNK(st_source.st_mode): sourcepath = os.path.join(source_root, relpath) linkto = os.path.join(os.path.dirname(relpath), os.readlink(sourcepath)) if linkto.startswith('/'): linkto = linkto[1:] linktarget = os.path.join(target_root, linkto) try: os.rmdir(linktarget) except OSError: pass if not os.path.exists(linktarget): try: os.makedirs(os.path.dirname(linktarget)) except OSError as e: if e.errno != errno.EEXIST: raise shutil.move(targetpath, linktarget) return # We're installing a non-directory over an existing non-empty directory, # and we have no better strategy. Move the existing directory to a # backup location. backuppath = targetpath + '.bak' while True: if not os.path.exists(backuppath): os.rename(targetpath, backuppath) break else: backuppath = backuppath + '.bak'
def copy_all(self): """Core copy process. This is the most important step of this stage. It clones live filesystem into a local partition in the selected hard disk.""" self.db.progress('START', 0, 100, 'ubiquity/install/title') self.db.progress('INFO', 'ubiquity/install/copying') fs_size = os.path.join(self.casper_path, 'filesystem.size') if os.path.exists(fs_size): with open(fs_size) as total_size_fp: total_size = int(total_size_fp.readline()) else: # Fallback in case an Ubuntu derivative forgets to put # /casper/filesystem.size on the CD, or to account for things # like CD->USB transformation tools that don't copy this file. # This is slower than just reading the size from a file, but # better than crashing. # # Obviously doing os.walk() twice is inefficient, but I'd rather # not suck the list into ubiquity's memory, and I'm guessing # that the kernel's dentry cache will avoid most of the slowness # anyway. total_size = 0 for dirpath, dirnames, filenames in os.walk(self.source): for name in dirnames + filenames: fqpath = os.path.join(dirpath, name) total_size += os.lstat(fqpath).st_size # Progress bar handling: # We sample progress every half-second (assuming time.time() gives # us sufficiently good granularity) and use the average of progress # over the last minute or so to decide how much time remains. We # don't bother displaying any progress for the first ten seconds in # order to allow things to settle down, and we only update the "time # remaining" indicator at most every two seconds after that. copy_progress = 0 copied_size = 0 directory_times = [] time_start = time.time() times = [(time_start, copied_size)] long_enough = False time_last_update = time_start debug = 'UBIQUITY_DEBUG' in os.environ if self.db.get('ubiquity/install/md5_check') == 'false': md5_check = False else: md5_check = True # Increase kernel flush times during bulk data copying to make it # more likely that small files are packed contiguously, which should # speed up initial boot times. dirty_writeback_centisecs = None dirty_expire_centisecs = None if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'): with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc: dirty_writeback_centisecs = int(dwc.readline()) with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print('3000\n', file=dwc) if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'): with open('/proc/sys/vm/dirty_expire_centisecs') as dec: dirty_expire_centisecs = int(dec.readline()) with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print('6000\n', file=dec) old_umask = os.umask(0) for dirpath, dirnames, filenames in os.walk(self.source): sp = dirpath[len(self.source) + 1:] for name in dirnames + filenames: relpath = os.path.join(sp, name) # /etc/fstab was legitimately created by partman, and # shouldn't be copied again. Similarly, /etc/crypttab may # have been legitimately created by the user-setup plugin. if relpath in ("etc/fstab", "etc/crypttab"): continue sourcepath = os.path.join(self.source, relpath) targetpath = os.path.join(self.target, relpath) st = os.lstat(sourcepath) # Is the path blacklisted? if (not stat.S_ISDIR(st.st_mode) and '/%s' % relpath in self.blacklist): if debug: syslog.syslog('Not copying %s' % relpath) continue # Remove the target if necessary and if we can. install_misc.remove_target(self.source, self.target, relpath, st) # Now actually copy source to target. mode = stat.S_IMODE(st.st_mode) if stat.S_ISLNK(st.st_mode): linkto = os.readlink(sourcepath) os.symlink(linkto, targetpath) elif stat.S_ISDIR(st.st_mode): if not os.path.isdir(targetpath): try: os.mkdir(targetpath, mode) except OSError as e: # there is a small window where update-apt-cache # can race with us since it creates # "/target/var/cache/apt/...". Hence, ignore # failure if the directory does now exist where # brief moments before it didn't. if e.errno != errno.EEXIST: raise elif stat.S_ISCHR(st.st_mode): os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev) elif stat.S_ISBLK(st.st_mode): os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev) elif stat.S_ISFIFO(st.st_mode): os.mknod(targetpath, stat.S_IFIFO | mode) elif stat.S_ISSOCK(st.st_mode): os.mknod(targetpath, stat.S_IFSOCK | mode) elif stat.S_ISREG(st.st_mode): install_misc.copy_file(self.db, sourcepath, targetpath, md5_check) # Copy metadata. copied_size += st.st_size os.lchown(targetpath, st.st_uid, st.st_gid) if not stat.S_ISLNK(st.st_mode): os.chmod(targetpath, mode) if stat.S_ISDIR(st.st_mode): directory_times.append( (targetpath, st.st_atime, st.st_mtime)) # os.utime() sets timestamp of target, not link elif not stat.S_ISLNK(st.st_mode): try: os.utime(targetpath, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass if (hasattr(os, "listxattr") and hasattr(os, "supports_follow_symlinks") and os.supports_follow_symlinks): try: attrnames = os.listxattr(sourcepath, follow_symlinks=False) for attrname in attrnames: attrvalue = os.getxattr(sourcepath, attrname, follow_symlinks=False) os.setxattr(targetpath, attrname, attrvalue, follow_symlinks=False) except OSError as e: if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA): raise if int((copied_size * 90) / total_size) != copy_progress: copy_progress = int((copied_size * 90) / total_size) self.db.progress('SET', 10 + copy_progress) time_now = time.time() if (time_now - times[-1][0]) >= 0.5: times.append((time_now, copied_size)) if not long_enough and time_now - times[0][0] >= 10: long_enough = True if long_enough and time_now - time_last_update >= 2: time_last_update = time_now while (time_now - times[0][0] > 60 and time_now - times[1][0] >= 60): times.pop(0) speed = ((times[-1][1] - times[0][1]) / (times[-1][0] - times[0][0])) if speed != 0: time_remaining = (int( (total_size - copied_size) / speed)) if time_remaining < 60: self.db.progress( 'INFO', 'ubiquity/install/copying_minute') # Apply timestamps to all directories now that the items within them # have been copied. for dirtime in directory_times: (directory, atime, mtime) = dirtime try: os.utime(directory, (atime, mtime)) except Exception: # I have no idea why I've been getting lots of bug reports # about this failing, but I really don't care. Ignore it. pass # Revert to previous kernel flush times. if dirty_writeback_centisecs is not None: with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print(dirty_writeback_centisecs, file=dwc) if dirty_expire_centisecs is not None: with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print(dirty_expire_centisecs, file=dec) # Try some possible locations for the kernel we used to boot. This # lets us save a couple of megabytes of CD space. bootdir = self.target_file('boot') kernel = self.find_cd_kernel() if kernel: prefix = os.path.basename(kernel).split('-', 1)[0] release = os.uname()[2] target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release)) copies = [] # ISO9660 images may have to use .efi rather than .efi.signed in # order to support being booted using isolinux, which must abide # by archaic 8.3 restrictions. for suffix in (".efi", ".efi.signed"): if os.path.exists(kernel + suffix): signed_kernel = kernel + suffix break else: signed_kernel = None if os.path.exists(kernel): copies.append((kernel, target_kernel)) elif signed_kernel is not None: # No unsigned kernel. We'll construct it using sbsigntool. copies.append((signed_kernel, target_kernel)) if signed_kernel is not None: copies.append((signed_kernel, "%s.efi.signed" % target_kernel)) for source, target in copies: osextras.unlink_force(target) install_misc.copy_file(self.db, source, target, md5_check) os.lchown(target, 0, 0) os.chmod(target, 0o644) st = os.lstat(source) try: os.utime(target, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass if not os.path.exists(kernel) and signed_kernel is not None: # Construct the unsigned kernel. subprocess.check_call(["sbattach", "--remove", target_kernel]) os.umask(old_umask) self.db.progress('SET', 100) self.db.progress('STOP')
def select_language_packs(self, save=False): try: keep_packages = self.db.get('ubiquity/keep-installed') keep_packages = keep_packages.replace(',', '').split() syslog.syslog('keeping packages due to preseeding: %s' % ' '.join(keep_packages)) record_installed(keep_packages) except debconf.DebconfError: pass langpacks = [] all_langpacks = False try: langpack_db = self.db.get('pkgsel/language-packs') if langpack_db == 'ALL': apt_out = subprocess.Popen( ['apt-cache', '-n', 'search', '^language-pack-[^-][^-]*$'], stdout=subprocess.PIPE).communicate()[0].rstrip().split('\n') langpacks = map(lambda x: x.split('-')[2].strip(), apt_out) all_langpacks = True else: langpacks = langpack_db.replace(',', '').split() except debconf.DebconfError: pass if not langpacks: langpack_set = set() try: langpack_db = self.db.get('localechooser/supported-locales') for locale in langpack_db.replace(',', '').split(): langpack_set.add(locale) except debconf.DebconfError: pass langpack_db = self.db.get('debian-installer/locale') langpack_set.add(langpack_db) langpacks = sorted(langpack_set) no_install = '/var/lib/ubiquity/no-install-langpacks' if os.path.exists(no_install): osextras.unlink_force(no_install) if len(langpacks) == 1 and langpacks[0] in ('C', 'en'): # Touch with open(no_install, 'a'): os.utime(no_install, None) syslog.syslog('keeping language packs for: %s' % ' '.join(langpacks)) try: lppatterns = self.db.get('pkgsel/language-pack-patterns').split() except debconf.DebconfError: return cache = Cache() to_install = [] checker = osextras.find_on_path('check-language-support') for lp_locale in langpacks: lp = locale_to_language_pack(lp_locale) # Basic language packs, required to get localisation working at # all. We install these almost unconditionally; if you want to # get rid of even these, you can preseed pkgsel/language-packs # to the empty string. to_install.append('language-pack-%s' % lp) # Other language packs, typically selected by preseeding. for pattern in lppatterns: to_install.append(pattern.replace('$LL', lp)) # More extensive language support packages. # If pkgsel/language-packs is ALL, then speed things up by # calling check-language-support just once. if not all_langpacks and checker: check_lang = subprocess.Popen( ['check-language-support', '-l', lp_locale.split('.')[0], '--show-installed'], stdout=subprocess.PIPE) to_install.extend(check_lang.communicate()[0].strip().split()) else: to_install.append('language-support-%s' % lp) if checker: # Keep language-support-$LL installed if it happens to be in # the live filesystem, since there's no point spending time # removing it; but don't install it if it isn't in the live # filesystem. toplevel = 'language-support-%s' % lp toplevel_pkg = get_cache_pkg(cache, toplevel) if toplevel_pkg and toplevel_pkg.is_installed: to_install.append(toplevel) if all_langpacks and checker: check_lang = subprocess.Popen( ['check-language-support', '-a', '--show-installed'], stdout=subprocess.PIPE) to_install.extend(check_lang.communicate()[0].strip().split()) # Filter the list of language packs to include only language packs # that exist in the live filesystem's apt cache, so that we can tell # the difference between "no such language pack" and "language pack # not retrievable given apt configuration in /target" later on. to_install = [lp for lp in to_install if get_cache_pkg(cache, lp) is not None] install_new = True try: install_new_key = \ self.db.get('pkgsel/install-language-support') == 'true' if install_new_key != '' and not misc.create_bool(install_new_key): install_new = False except debconf.DebconfError: pass if not install_new: # Keep packages that are on the live filesystem, but don't install # new ones. # TODO cjwatson 2010-03-18: To match pkgsel's semantics, we ought to # be willing to install packages from the package pool on the CD as # well. to_install = [lp for lp in to_install if get_cache_pkg(cache, lp).is_installed] del cache record_installed(to_install) langpacks_file = '/var/lib/ubiquity/langpacks' if os.path.exists(langpacks_file): osextras.unlink_force(langpacks_file) if install_new: if save: if not os.path.exists(os.path.dirname(langpacks_file)): os.makedirs(os.path.dirname(langpacks_file)) with open(langpacks_file, 'w') as langpacks: for pkg in to_install: print >>langpacks, pkg return [] else: return to_install
def remove_target(source_root, target_root, relpath, st_source): """Remove a target file if necessary and if we can. On the whole, we can assume that partman-target has arranged to clear out the areas of the filesystem we're installing to. However, in edge cases it's possible that there is still some detritus left over, and we want to steer a reasonable course between cavalierly destroying data and crashing. So, we remove non-directories and empty directories that are in our way, but if a non-empty directory is in our way then we move it aside (adding .bak suffixes until we find something unused) instead. """ targetpath = os.path.join(target_root, relpath) try: st_target = os.lstat(targetpath) except OSError: # The target does not exist. Boring. return if stat.S_ISDIR(st_source.st_mode) and stat.S_ISDIR(st_target.st_mode): # One directory is as good as another, so we don't need to remove an # existing directory just in order to create another one. return if not stat.S_ISDIR(st_target.st_mode): # Installing over a non-directory is easy; just remove it. osextras.unlink_force(targetpath) return try: # Is it an empty directory? That's easy too. os.rmdir(targetpath) return except OSError as e: if e.errno not in (errno.ENOTEMPTY, errno.EEXIST): raise # If we've got this far, then we must be trying to install a # non-directory over an existing non-empty directory. The slightly # easier case is if it's a symlink, and if the prospective symlink # target hasn't been copied yet or is empty; in that case, we should try # to move the existing directory to the symlink target. if stat.S_ISLNK(st_source.st_mode): sourcepath = os.path.join(source_root, relpath) linkto = os.path.join( os.path.dirname(relpath), os.readlink(sourcepath)) if linkto.startswith('/'): linkto = linkto[1:] linktarget = os.path.join(target_root, linkto) try: os.rmdir(linktarget) except OSError: pass if not os.path.exists(linktarget): try: os.makedirs(os.path.dirname(linktarget)) except OSError as e: if e.errno != errno.EEXIST: raise shutil.move(targetpath, linktarget) return # We're installing a non-directory over an existing non-empty directory, # and we have no better strategy. Move the existing directory to a # backup location. backuppath = targetpath + '.bak' while True: if not os.path.exists(backuppath): os.rename(targetpath, backuppath) break else: backuppath = backuppath + '.bak'
def main(oem_config): force_utf8_locale() usage = '%prog [options] [frontend]' parser = optparse.OptionParser(usage=usage, version=VERSION) parser.set_defaults( debug=('UBIQUITY_DEBUG' in os.environ), debug_pdb=False, cdebconf=False, automatic=False, query=False) parser.add_option('-d', '--debug', dest='debug', action='store_true', help='debug mode (warning: passwords will be logged!)') parser.add_option('--pdb', dest='debug_pdb', action='store_true', help='drop into Python debugger on a crash') parser.add_option('--cdebconf', dest='cdebconf', action='store_true', help='use cdebconf instead of debconf (experimental)') parser.add_option('--automatic', dest='automatic', action='store_true', help='do not ignore the "seen" flag (useful for ' 'unattended installations).') parser.add_option('--only', dest='only', action='store_true', help='tell the application that it is the only desktop ' 'program running so that it can customize its UI ' 'to better suit a minimal environment.') parser.add_option('-q', '--query', dest='query', action='store_true', help='find out which frontend will be used by default') parser.add_option('-g', '--greeter', dest='greeter', action='store_true', help='allow the user to leave the installer and enter ' 'a live desktop (for the initial boot).') parser.add_option('-b', '--no-bootloader', dest='bootloader', default=True, action='store_false', help='Do not install a bootloader.') parser.add_option('--ldtp', dest='ldtp', action='store_true', help='Name widgets in ATK by their GtkBuilder names, ' 'to support LDTP testing.') parser.add_option('--no-webcam', dest='webcam', default=True, action='store_false', help='Disable the webcam page.') (options, args) = parser.parse_args() if options.debug: os.environ['UBIQUITY_DEBUG'] = '1' if options.debug_pdb: os.environ['UBIQUITY_DEBUG_PDB'] = '1' if options.cdebconf: # Note that this needs to be set before DebconfCommunicate is # imported by anything. os.environ['DEBCONF_USE_CDEBCONF'] = '1' prepend_path('/usr/lib/cdebconf') prepend_path('/usr/lib/ubiquity/compat') if options.automatic: os.environ['UBIQUITY_AUTOMATIC'] = '1' if options.greeter: os.environ['UBIQUITY_GREETER'] = '1' if options.only: os.environ['UBIQUITY_ONLY'] = '1' if not options.bootloader: os.environ['UBIQUITY_NO_BOOTLOADER'] = '1' if oem_config: os.environ['UBIQUITY_OEM_USER_CONFIG'] = '1' global TARGET TARGET = '' if options.ldtp: os.environ['UBIQUITY_LDTP'] = '1' if not options.webcam: os.environ['UBIQUITY_NO_WEBCAM'] = '1' acquire_lock() try: os.makedirs('/var/log/installer') except OSError as e: # be happy if someone already created the path if e.errno != errno.EEXIST: raise syslog.openlog('ubiquity', syslog.LOG_NOWAIT | syslog.LOG_PID) if oem_config: syslog.syslog("Ubiquity %s (oem-config)" % VERSION) else: syslog.syslog("Ubiquity %s" % VERSION) with open('/var/log/installer/version', 'w') as version_file: print('ubiquity %s' % VERSION, file=version_file) if 'UBIQUITY_DEBUG' in os.environ: if 'UBIQUITY_DEBUG_CORE' not in os.environ: os.environ['UBIQUITY_DEBUG_CORE'] = '1' if 'DEBCONF_DEBUG' not in os.environ: os.environ['DEBCONF_DEBUG'] = 'developer' # The frontend should take care of displaying a helpful message if # we are being run without root privileges. if not (options.query and args and args[0] == 'noninteractive'): try: if oem_config: logfile = '/var/log/oem-config.log' else: logfile = '/var/log/installer/debug' log = os.open(logfile, os.O_WRONLY | os.O_CREAT | os.O_APPEND, 0o644) os.dup2(log, 2) os.close(log) sys.stderr = os.fdopen(2, 'a', 1) if oem_config: print("Ubiquity %s (oem-config)" % VERSION, file=sys.stderr) else: print("Ubiquity %s" % VERSION, file=sys.stderr) except (IOError, OSError) as err: if err.errno != errno.EACCES: raise # Default to enabling internal (non-debconf) debugging except for when # using --automatic. if 'UBIQUITY_DEBUG_CORE' not in os.environ: if options.automatic: os.environ['UBIQUITY_DEBUG_CORE'] = '1' # Initialise cdebconf if necessary, to work around # http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=451130. if options.cdebconf: for package in ('ubiquity', 'keyboard-configuration', 'console-setup'): subprocess.call(['debconf-loadtemplate', package, '/var/lib/dpkg/info/%s.templates' % package]) # Clean up old state. for name in ('apt-installed', 'apt-install-direct', 'remove-kernels', 'apt-removed', 'encrypted-swap', 'started-installing'): osextras.unlink_force(os.path.join('/var/lib/ubiquity', name)) shutil.rmtree("/var/lib/partman", ignore_errors=True) misc.remove_os_prober_cache() if oem_config and not options.query: disable_autologin() if args: install(args[0], query=options.query) else: install(query=options.query) if oem_config: run_oem_hooks() im_switch.kill_im()
def select_ecryptfs(self): """Is ecryptfs in use by an existing user? If so, keep it installed. This duplicates code from user-setup, but necessarily so; when user-setup-ask runs in ubiquity, /target is not yet mounted, but we need to make this decision before generating the file copy blacklist so user-setup-apply would be too late.""" home = self.target_file('home') if os.path.isdir(home): for homedir in os.listdir(home): if os.path.isdir(os.path.join(home, homedir, '.ecryptfs')): syslog.syslog('ecryptfs already in use in %s' % os.path.join(home, homedir)) install_misc.record_installed(['ecryptfs-utils']) break if __name__ == '__main__': os.environ['DPKG_UNTRANSLATED_MESSAGES'] = '1' if not os.path.exists('/var/lib/ubiquity'): os.makedirs('/var/lib/ubiquity') osextras.unlink_force('/var/lib/ubiquity/install.trace') install = Install() sys.excepthook = install_misc.excepthook install.run() sys.exit(0) # vim:ai:et:sts=4:tw=80:sw=4:
if not os.path.isdir(targetpath): os.mkdir(targetpath, mode) elif stat.S_ISCHR(st.st_mode): os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev) elif stat.S_ISBLK(st.st_mode): os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev) elif stat.S_ISFIFO(st.st_mode): os.mknod(targetpath, stat.S_IFIFO | mode) elif stat.S_ISSOCK(st.st_mode): os.mknod(targetpath, stat.S_IFSOCK | mode) elif stat.S_ISREG(st.st_mode): if '/%s' % relpath in self.blacklist: if debug: syslog.syslog('Not copying %s' % relpath) continue osextras.unlink_force(targetpath) install_misc.copy_file(self.db, sourcepath, targetpath, md5_check) copied_size += st.st_size os.lchown(targetpath, st.st_uid, st.st_gid) if not stat.S_ISLNK(st.st_mode): os.chmod(targetpath, mode) if stat.S_ISDIR(st.st_mode): directory_times.append((targetpath, st.st_atime, st.st_mtime)) # os.utime() sets timestamp of target, not link elif not stat.S_ISLNK(st.st_mode): os.utime(targetpath, (st.st_atime, st.st_mtime)) if int((copied_size * 90) / total_size) != copy_progress: copy_progress = int((copied_size * 90) / total_size) self.db.progress('SET', 10 + copy_progress)
def test_unlink_force_unlinks_existing(self): path = os.path.join(self.temp_dir, "file") self.create_empty_file(path) self.assertTrue(os.path.exists(path)) osextras.unlink_force(path) self.assertFalse(os.path.exists(path))
def commit_with_verify(self, cache, fetch_progress, install_progress): # Hack around occasional undetected download errors in apt by doing # our own verification pass at the end. See # https://bugs.launchpad.net/bugs/922949. Unfortunately this means # clone-and-hacking most of cache.commit ... pm = apt_pkg.PackageManager(cache._depcache) fetcher = apt_pkg.Acquire(fetch_progress) while True: # fetch archives first res = cache._fetch_archives(fetcher, pm) # manually verify all the downloads syslog.syslog('Verifying downloads ...') for item in fetcher.items: with open(item.destfile, 'rb') as destfile: st = os.fstat(destfile.fileno()) if st.st_size != item.filesize: osextras.unlink_force(item.destfile) raise IOError( "%s size mismatch: %ld != %ld" % (item.destfile, st.st_size, item.filesize)) # Mapping back to the package object is an utter pain. # If we fail to find one, it's entirely possible it's a # programming error and not a download error, so skip # verification in such cases rather than failing. destfile_base = os.path.basename(item.destfile) try: name, version, arch = destfile_base.split('_') version = version.replace('%3a', ':') arch = arch.split('.')[0] if arch == 'all': fullname = name else: fullname = '%s:%s' % (name, arch) # This syntax only works on systems configured # for multiarch, so check and fall back to the # single-architecture syntax. if fullname not in cache: fullname = name candidate = cache[fullname].versions[version] except (KeyError, ValueError) as e: syslog.syslog( 'Failed to find package object for %s: %s' % (item.destfile, e)) continue if candidate.sha256 is not None: sha256 = hashlib.sha256() for chunk in iter(lambda: destfile.read(16384), b''): sha256.update(chunk) if sha256.hexdigest() != candidate.sha256: osextras.unlink_force(item.destfile) raise IOError( "%s SHA256 checksum mismatch: %s != %s" % (item.destfile, sha256.hexdigest(), candidate.sha256)) syslog.syslog('Downloads verified successfully') # then install res = cache.install_archives(pm, install_progress) if res == pm.RESULT_COMPLETED: break elif res == pm.RESULT_FAILED: raise SystemError("installArchives() failed") elif res == pm.RESULT_INCOMPLETE: pass else: raise SystemError("internal-error: unknown result code " "from InstallArchives: %s" % res) # reload the fetcher for media swapping fetcher.shutdown() return (res == pm.RESULT_COMPLETED)
def select_language_packs(self, save=False): try: keep_packages = self.db.get('ubiquity/keep-installed') keep_packages = keep_packages.replace(',', '').split() syslog.syslog('keeping packages due to preseeding: %s' % ' '.join(keep_packages)) record_installed(keep_packages) except debconf.DebconfError: pass langpacks = [] all_langpacks = False try: langpack_db = self.db.get('pkgsel/language-packs') if langpack_db == 'ALL': apt_subp = subprocess.Popen( ['apt-cache', '-n', 'search', '^language-pack-[^-][^-]*$'], stdout=subprocess.PIPE, universal_newlines=True) apt_out = apt_subp.communicate()[0].rstrip().split('\n') langpacks = [x.split('-')[2].strip() for x in apt_out] all_langpacks = True else: langpacks = langpack_db.replace(',', '').split() except debconf.DebconfError: pass if not langpacks: langpack_set = set() try: langpack_db = self.db.get('localechooser/supported-locales') for locale in langpack_db.replace(',', '').split(): langpack_set.add(locale) except debconf.DebconfError: pass langpack_db = self.db.get('debian-installer/locale') langpack_set.add(langpack_db) langpacks = sorted(langpack_set) no_install = '/var/lib/ubiquity/no-install-langpacks' if os.path.exists(no_install): osextras.unlink_force(no_install) if len(langpacks) == 1 and langpacks[0] in ('C', 'en'): # Touch with open(no_install, 'a'): os.utime(no_install, None) syslog.syslog('keeping language packs for: %s' % ' '.join(langpacks)) try: lppatterns = self.db.get('pkgsel/language-pack-patterns').split() except debconf.DebconfError: return cache = Cache() to_install = [] checker = osextras.find_on_path('check-language-support') for lp_locale in langpacks: lp = locale_to_language_pack(lp_locale) # Basic language packs, required to get localisation working at # all. We install these almost unconditionally; if you want to # get rid of even these, you can preseed pkgsel/language-packs # to the empty string. to_install.append('language-pack-%s' % lp) # Other language packs, typically selected by preseeding. for pattern in lppatterns: to_install.append(pattern.replace('$LL', lp)) # More extensive language support packages. # If pkgsel/language-packs is ALL, then speed things up by # calling check-language-support just once. if not all_langpacks and checker: check_lang = subprocess.Popen([ 'check-language-support', '-l', lp_locale.split('.')[0], '--show-installed' ], stdout=subprocess.PIPE, universal_newlines=True) to_install.extend(check_lang.communicate()[0].strip().split()) else: to_install.append('language-support-%s' % lp) if checker: # Keep language-support-$LL installed if it happens to be in # the live filesystem, since there's no point spending time # removing it; but don't install it if it isn't in the live # filesystem. toplevel = 'language-support-%s' % lp toplevel_pkg = get_cache_pkg(cache, toplevel) if toplevel_pkg and toplevel_pkg.is_installed: to_install.append(toplevel) if all_langpacks and checker: check_lang = subprocess.Popen( ['check-language-support', '-a', '--show-installed'], stdout=subprocess.PIPE, universal_newlines=True) to_install.extend(check_lang.communicate()[0].strip().split()) # Filter the list of language packs to include only language packs # that exist in the live filesystem's apt cache, so that we can tell # the difference between "no such language pack" and "language pack # not retrievable given apt configuration in /target" later on. to_install = [ pkg for pkg in to_install if get_cache_pkg(cache, pkg) is not None ] install_new = True try: install_new_key = \ self.db.get('pkgsel/install-language-support') == 'true' if install_new_key != '' and not misc.create_bool(install_new_key): install_new = False except debconf.DebconfError: pass if not install_new: # Keep packages that are on the live filesystem, but don't install # new ones. # TODO cjwatson 2010-03-18: To match pkgsel's semantics, we # ought to be willing to install packages from the package pool # on the CD as well. to_install = [ pkg for pkg in to_install if get_cache_pkg(cache, pkg).is_installed ] del cache record_installed(to_install) langpacks_file = '/var/lib/ubiquity/langpacks' if os.path.exists(langpacks_file): osextras.unlink_force(langpacks_file) if install_new: if save: if not os.path.exists(os.path.dirname(langpacks_file)): os.makedirs(os.path.dirname(langpacks_file)) with open(langpacks_file, 'w') as langpacks: for pkg in to_install: print(pkg, file=langpacks) return [] else: return to_install
def copy_all(self): """Core copy process. This is the most important step of this stage. It clones live filesystem into a local partition in the selected hard disk.""" self.db.progress('START', 0, 100, 'ubiquity/install/title') self.db.progress('INFO', 'ubiquity/install/copying') fs_size = os.path.join(self.casper_path, 'filesystem.size') if os.path.exists(fs_size): with open(fs_size) as total_size_fp: total_size = int(total_size_fp.readline()) else: # Fallback in case an Linux Mint derivative forgets to put # /casper/filesystem.size on the CD, or to account for things # like CD->USB transformation tools that don't copy this file. # This is slower than just reading the size from a file, but # better than crashing. # # Obviously doing os.walk() twice is inefficient, but I'd rather # not suck the list into ubiquity's memory, and I'm guessing # that the kernel's dentry cache will avoid most of the slowness # anyway. total_size = 0 for dirpath, dirnames, filenames in os.walk(self.source): for name in dirnames + filenames: fqpath = os.path.join(dirpath, name) total_size += os.lstat(fqpath).st_size # Progress bar handling: # We sample progress every half-second (assuming time.time() gives # us sufficiently good granularity) and use the average of progress # over the last minute or so to decide how much time remains. We # don't bother displaying any progress for the first ten seconds in # order to allow things to settle down, and we only update the "time # remaining" indicator at most every two seconds after that. copy_progress = 0 copied_size = 0 directory_times = [] time_start = time.time() times = [(time_start, copied_size)] long_enough = False time_last_update = time_start debug = 'UBIQUITY_DEBUG' in os.environ if self.db.get('ubiquity/install/md5_check') == 'false': md5_check = False else: md5_check = True # Increase kernel flush times during bulk data copying to make it # more likely that small files are packed contiguously, which should # speed up initial boot times. dirty_writeback_centisecs = None dirty_expire_centisecs = None if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'): with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc: dirty_writeback_centisecs = int(dwc.readline()) with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print('3000\n', file=dwc) if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'): with open('/proc/sys/vm/dirty_expire_centisecs') as dec: dirty_expire_centisecs = int(dec.readline()) with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print('6000\n', file=dec) old_umask = os.umask(0) for dirpath, dirnames, filenames in os.walk(self.source): sp = dirpath[len(self.source) + 1:] for name in dirnames + filenames: relpath = os.path.join(sp, name) # /etc/fstab was legitimately created by partman, and # shouldn't be copied again. Similarly, /etc/crypttab may # have been legitimately created by the user-setup plugin. if relpath in ("etc/fstab", "etc/crypttab"): continue sourcepath = os.path.join(self.source, relpath) targetpath = os.path.join(self.target, relpath) st = os.lstat(sourcepath) # Is the path blacklisted? if (not stat.S_ISDIR(st.st_mode) and '/%s' % relpath in self.blacklist): if debug: syslog.syslog('Not copying %s' % relpath) continue # Remove the target if necessary and if we can. install_misc.remove_target( self.source, self.target, relpath, st) # Now actually copy source to target. mode = stat.S_IMODE(st.st_mode) if stat.S_ISLNK(st.st_mode): linkto = os.readlink(sourcepath) os.symlink(linkto, targetpath) elif stat.S_ISDIR(st.st_mode): if not os.path.isdir(targetpath): try: os.mkdir(targetpath, mode) except OSError as e: # there is a small window where update-apt-cache # can race with us since it creates # "/target/var/cache/apt/...". Hence, ignore # failure if the directory does now exist where # brief moments before it didn't. if e.errno != errno.EEXIST: raise elif stat.S_ISCHR(st.st_mode): os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev) elif stat.S_ISBLK(st.st_mode): os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev) elif stat.S_ISFIFO(st.st_mode): os.mknod(targetpath, stat.S_IFIFO | mode) elif stat.S_ISSOCK(st.st_mode): os.mknod(targetpath, stat.S_IFSOCK | mode) elif stat.S_ISREG(st.st_mode): install_misc.copy_file( self.db, sourcepath, targetpath, md5_check) # Copy metadata. copied_size += st.st_size os.lchown(targetpath, st.st_uid, st.st_gid) if not stat.S_ISLNK(st.st_mode): os.chmod(targetpath, mode) if stat.S_ISDIR(st.st_mode): directory_times.append( (targetpath, st.st_atime, st.st_mtime)) # os.utime() sets timestamp of target, not link elif not stat.S_ISLNK(st.st_mode): try: os.utime(targetpath, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass if (hasattr(os, "listxattr") and hasattr(os, "supports_follow_symlinks") and os.supports_follow_symlinks): try: attrnames = os.listxattr( sourcepath, follow_symlinks=False) for attrname in attrnames: attrvalue = os.getxattr( sourcepath, attrname, follow_symlinks=False) os.setxattr( targetpath, attrname, attrvalue, follow_symlinks=False) except OSError as e: if e.errno not in ( errno.EPERM, errno.ENOTSUP, errno.ENODATA): raise if int((copied_size * 90) / total_size) != copy_progress: copy_progress = int((copied_size * 90) / total_size) self.db.progress('SET', 10 + copy_progress) time_now = time.time() if (time_now - times[-1][0]) >= 0.5: times.append((time_now, copied_size)) if not long_enough and time_now - times[0][0] >= 10: long_enough = True if long_enough and time_now - time_last_update >= 2: time_last_update = time_now while (time_now - times[0][0] > 60 and time_now - times[1][0] >= 60): times.pop(0) speed = ((times[-1][1] - times[0][1]) / (times[-1][0] - times[0][0])) if speed != 0: time_remaining = ( int((total_size - copied_size) / speed)) if time_remaining < 60: self.db.progress( 'INFO', 'ubiquity/install/copying_minute') # Apply timestamps to all directories now that the items within them # have been copied. for dirtime in directory_times: (directory, atime, mtime) = dirtime try: os.utime(directory, (atime, mtime)) except Exception: # I have no idea why I've been getting lots of bug reports # about this failing, but I really don't care. Ignore it. pass # Revert to previous kernel flush times. if dirty_writeback_centisecs is not None: with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print(dirty_writeback_centisecs, file=dwc) if dirty_expire_centisecs is not None: with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print(dirty_expire_centisecs, file=dec) # Try some possible locations for the kernel we used to boot. This # lets us save a couple of megabytes of CD space. bootdir = self.target_file('boot') kernel = self.find_cd_kernel() if kernel: prefix = os.path.basename(kernel).split('-', 1)[0] release = os.uname()[2] target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release)) copies = [] # ISO9660 images may have to use .efi rather than .efi.signed in # order to support being booted using isolinux, which must abide # by archaic 8.3 restrictions. for suffix in (".efi", ".efi.signed"): if os.path.exists(kernel + suffix): signed_kernel = kernel + suffix break else: signed_kernel = None if os.path.exists(kernel): copies.append((kernel, target_kernel)) elif signed_kernel is not None: # No unsigned kernel. We'll construct it using sbsigntool. copies.append((signed_kernel, target_kernel)) if signed_kernel is not None: copies.append((signed_kernel, "%s.efi.signed" % target_kernel)) for source, target in copies: osextras.unlink_force(target) install_misc.copy_file(self.db, source, target, md5_check) os.lchown(target, 0, 0) os.chmod(target, 0o644) st = os.lstat(source) try: os.utime(target, (st.st_atime, st.st_mtime)) except Exception: # We can live with timestamps being wrong. pass if not os.path.exists(kernel) and signed_kernel is not None: # Construct the unsigned kernel. subprocess.check_call(["sbattach", "--remove", target_kernel]) os.umask(old_umask) self.db.progress('SET', 100) self.db.progress('STOP')
def copy_all(self): """Core copy process. This is the most important step of this stage. It clones live filesystem into a local partition in the selected hard disk.""" self.db.progress('START', 0, 100, 'ubiquity/install/title') self.db.progress('INFO', 'ubiquity/install/copying') fs_size = os.path.join(self.casper_path, 'filesystem.size') assert os.path.exists(fs_size), "Missing filesystem.size." with open(fs_size) as total_size_fp: total_size = int(total_size_fp.readline()) # Progress bar handling: # We sample progress every half-second (assuming time.time() gives # us sufficiently good granularity) and use the average of progress # over the last minute or so to decide how much time remains. We # don't bother displaying any progress for the first ten seconds in # order to allow things to settle down, and we only update the "time # remaining" indicator at most every two seconds after that. copy_progress = 0 copied_size, counter = 0, 0 directory_times = [] time_start = time.time() times = [(time_start, copied_size)] long_enough = False time_last_update = time_start debug = 'UBIQUITY_DEBUG' in os.environ if self.db.get('ubiquity/install/md5_check') == 'false': md5_check = False else: md5_check = True # Increase kernel flush times during bulk data copying to make it # more likely that small files are packed contiguously, which should # speed up initial boot times. dirty_writeback_centisecs = None dirty_expire_centisecs = None if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'): with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc: dirty_writeback_centisecs = int(dwc.readline()) with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print >>dwc, '3000\n' if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'): with open('/proc/sys/vm/dirty_expire_centisecs') as dec: dirty_expire_centisecs = int(dec.readline()) with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print >>dec, '6000\n' old_umask = os.umask(0) for dirpath, dirnames, filenames in os.walk(self.source): sp = dirpath[len(self.source) + 1:] for name in dirnames + filenames: relpath = os.path.join(sp, name) # /etc/fstab was legitimately created by partman, and # shouldn't be copied again. if relpath == "etc/fstab": continue sourcepath = os.path.join(self.source, relpath) targetpath = os.path.join(self.target, relpath) st = os.lstat(sourcepath) mode = stat.S_IMODE(st.st_mode) if stat.S_ISLNK(st.st_mode): if os.path.lexists(targetpath): os.unlink(targetpath) linkto = os.readlink(sourcepath) os.symlink(linkto, targetpath) elif stat.S_ISDIR(st.st_mode): if not os.path.isdir(targetpath): os.mkdir(targetpath, mode) elif stat.S_ISCHR(st.st_mode): os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev) elif stat.S_ISBLK(st.st_mode): os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev) elif stat.S_ISFIFO(st.st_mode): os.mknod(targetpath, stat.S_IFIFO | mode) elif stat.S_ISSOCK(st.st_mode): os.mknod(targetpath, stat.S_IFSOCK | mode) elif stat.S_ISREG(st.st_mode): if '/%s' % relpath in self.blacklist: if debug: syslog.syslog('Not copying %s' % relpath) continue osextras.unlink_force(targetpath) install_misc.copy_file(self.db, sourcepath, targetpath, md5_check) copied_size += st.st_size os.lchown(targetpath, st.st_uid, st.st_gid) if not stat.S_ISLNK(st.st_mode): os.chmod(targetpath, mode) if stat.S_ISDIR(st.st_mode): directory_times.append((targetpath, st.st_atime, st.st_mtime)) # os.utime() sets timestamp of target, not link elif not stat.S_ISLNK(st.st_mode): os.utime(targetpath, (st.st_atime, st.st_mtime)) if int((copied_size * 90) / total_size) != copy_progress: copy_progress = int((copied_size * 90) / total_size) self.db.progress('SET', 10 + copy_progress) time_now = time.time() if (time_now - times[-1][0]) >= 0.5: times.append((time_now, copied_size)) if not long_enough and time_now - times[0][0] >= 10: long_enough = True if long_enough and time_now - time_last_update >= 2: time_last_update = time_now while (time_now - times[0][0] > 60 and time_now - times[1][0] >= 60): times.pop(0) speed = ((times[-1][1] - times[0][1]) / (times[-1][0] - times[0][0])) if speed != 0: time_remaining = int((total_size - copied_size) / speed) if time_remaining < 60: self.db.progress( 'INFO', 'ubiquity/install/copying_minute') # Apply timestamps to all directories now that the items within them # have been copied. for dirtime in directory_times: (directory, atime, mtime) = dirtime try: os.utime(directory, (atime, mtime)) except OSError: # I have no idea why I've been getting lots of bug reports # about this failing, but I really don't care. Ignore it. pass # Revert to previous kernel flush times. if dirty_writeback_centisecs is not None: with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc: print >>dwc, dirty_writeback_centisecs if dirty_expire_centisecs is not None: with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec: print >>dec, dirty_expire_centisecs # Try some possible locations for the kernel we used to boot. This # lets us save a couple of megabytes of CD space. bootdir = os.path.join(self.target, 'boot') kernel = self.find_cd_kernel() if kernel: prefix = os.path.basename(kernel).split('-', 1)[0] release = os.uname()[2] target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release)) osextras.unlink_force(target_kernel) install_misc.copy_file(self.db, kernel, target_kernel, md5_check) os.lchown(target_kernel, 0, 0) os.chmod(target_kernel, 0644) st = os.lstat(kernel) os.utime(target_kernel, (st.st_atime, st.st_mtime)) os.umask(old_umask) self.db.progress('SET', 100) self.db.progress('STOP')
def commit_with_verify(self, cache, fetch_progress, install_progress): # Hack around occasional undetected download errors in apt by doing # our own verification pass at the end. See # https://bugs.launchpad.net/bugs/922949. Unfortunately this means # clone-and-hacking most of cache.commit ... pm = apt_pkg.PackageManager(cache._depcache) fetcher = apt_pkg.Acquire(fetch_progress) while True: # fetch archives first res = cache._fetch_archives(fetcher, pm) # manually verify all the downloads syslog.syslog('Verifying downloads ...') for item in fetcher.items: with open(item.destfile) as destfile: st = os.fstat(destfile.fileno()) if st.st_size != item.filesize: osextras.unlink_force(item.destfile) raise IOError( "%s size mismatch: %ld != %ld" % (item.destfile, st.st_size, item.filesize)) # Mapping back to the package object is an utter pain. # If we fail to find one, it's entirely possible it's a # programming error and not a download error, so skip # verification in such cases rather than failing. destfile_base = os.path.basename(item.destfile) try: name, version, arch = destfile_base.split('_') version = version.replace('%3a', ':') arch = arch.split('.')[0] if arch == 'all': fullname = name else: fullname = '%s:%s' % (name, arch) candidate = cache[fullname].versions[version] except (KeyError, ValueError), e: syslog.syslog( 'Failed to find package object for %s: %s' % (item.destfile, e)) continue if candidate.sha256 is not None: sha256 = hashlib.sha256() for chunk in iter(lambda: destfile.read(16384), b''): sha256.update(chunk) if sha256.hexdigest() != candidate.sha256: osextras.unlink_force(item.destfile) raise IOError( "%s SHA256 checksum mismatch: %s != %s" % (item.destfile, sha256.hexdigest(), candidate.sha256)) syslog.syslog('Downloads verified successfully') # then install res = cache.install_archives(pm, install_progress) if res == pm.RESULT_COMPLETED: break elif res == pm.RESULT_FAILED: raise SystemError("installArchives() failed") elif res == pm.RESULT_INCOMPLETE: pass else: raise SystemError("internal-error: unknown result code " "from InstallArchives: %s" % res) # reload the fetcher for media swapping fetcher.shutdown()