def get_apt_info(self, apt_sources_list): """Get apt information for an apt sources.list. Retrieves relevant information (with caching), and returns a tuple consisting of: latest version number, changelog. FIXME: has some trouble now with downgrade, maybe need to nuke /var/lib/apt/lists/vpnease* or something. Also, could we download lists to some temporary file instead? """ try: now = datetime.datetime.utcnow() # serve from cache? if self.cache.has_key(apt_sources_list): cinfo = self.cache[apt_sources_list] diff = now - cinfo.cachetime if (diff >= datetime.timedelta(0, 0, 0)) and (diff <= datetime.timedelta(0, self.interval, 0)): _log.info('serving cached apt info for version %s' % cinfo.version) return cinfo.version, cinfo.changelog else: pass # no, fetch using apt and cache _log.info('fetching apt info, apt source:') _log.info(apt_sources_list) helpers.write_file(self.tmpsource, apt_sources_list, perms=0644) run_command(['aptitude'] + self.aptitude_options + ['update'], retval=runcommand.FAIL) run_command(['aptitude'] + self.aptitude_options + ['download', 'vpnease'], cwd='/tmp', retval=runcommand.FAIL) version = None for i in os.listdir('/tmp'): if self.package_name_re.match(i): version = versioninfo.get_package_version_info(os.path.join('/tmp', i)) changelog = versioninfo.get_package_changelog(os.path.join('/tmp', i)) run_command(['/bin/rm', '-f', os.path.join('/tmp', i)], retval=runcommand.FAIL) if version is None: raise Exception('vpnease package file not found') if changelog is None: raise Exception('vpnease package changelog not found') # create a new cache object cinfo = AptCacheInfo(now, version, changelog) self.cache[apt_sources_list] = cinfo # return fresh data _log.info('serving fresh apt info for version %s' % cinfo.version) return cinfo.version, cinfo.changelog except: _log.exception('failed to get package version info') raise raise Exception('should not be here')
def run_update(self): # Ensure no questions asked os.environ['DEBIAN_FRONTEND'] = 'noninteractive' # XXX: is this ok? # os.environ['DEBIAN_PRIORITY'] = 'critical' # os.environ['DEBCONF_NOWARNINGS'] = 'yes' result = None old_version = None old_version_cached = None new_version = None new_version_cached = None # Tolerate logger errors try: from codebay.common import logger _log = logger.get('l2tpserver.update.update') except: _log = None try: from codebay.l2tpserver import constants from codebay.l2tpserver import runcommand from codebay.l2tpserver import versioninfo from codebay.l2tpserver import helpers run_command = runcommand.run_command except: print 'failed to import helpers, cannot update' return 3 try: def _run_aptitude(options, cwd=None): """Run aptitude with automatic -q and -y options.""" # XXX: add some option to make aptitude return with error instead of asking to ignore trust violations => may not be possible # - if cannot be done, missing repository keys will result in update timeout (aptitude will block) cmd = [constants.CMD_APTITUDE, '-q', '-y'] + options if _log is not None: _log.info('running: %s' % ' '.join(cmd)) retval, stdout, stderr = run_command(cmd, cwd=cwd) if retval != 0: if _log is not None: _log.warning('aptitude return value: %d' % retval) _log.warning('stdout:\n%s' % stdout) _log.warning('stderr:\n%s' % stderr) return retval, stdout, stderr def _dpkg_configure_fixup(): """Attempt 'dpkg --configure -a' fixup.""" # XXX: the previous worry here is that something may block, and an update # might actually fix the situation anyway. try: # XXX: more logging, XXX: path use [rv, out, err] = run_command([ 'dpkg', '--force-depends', '--force-confmiss', '--force-confdef', '--force-confold', '--configure', '-a' ], retval=runcommand.FAIL) except: if _log is not None: _log.exception( 'dpkg --configure -a failed, this is bad') def _cleanup_old_package_lists(): """Cleanup old package lists.""" try: if _log is not None: _log.info('cleaning up apt sources lists') # Remove all files in partial dir partial_dir = '/var/lib/apt/lists/partial' for f in os.listdir(partial_dir): fpath = os.path.join(partial_dir, f) if os.path.isfile(fpath): run_command(['/bin/rm', '-f', fpath]) # Remove all files except lock lists_dir = '/var/lib/apt/lists' for f in os.listdir(lists_dir): fpath = os.path.join(lists_dir, f) m = lockfile_name_re.match(f) if (os.path.isfile(fpath)) and (m is None): run_command(['/bin/rm', '-f', fpath]) except: if _log is not None: _log.exception('failed to remove package list files') def _aptitude_force_install(): """Attempt aptitude install which may clear up some (but not nearly all) error situations.""" try: # XXX: more logging here # XXX: add -q (quiet) option? run_command( [constants.CMD_APTITUDE, '-f', '-y', 'install'], retval=runcommand.FAIL) except: if _log is not None: _log.exception('apt-get cleanup failed, ignoring') # Figure out version v_name_re = re.compile('vpnease_.*\.deb') lockfile_name_re = re.compile('^lock$') old_version, old_version_cached = versioninfo.get_version_info() if _log is not None: _log.info('starting update, current_version: %s (cached: %s)' % (str(old_version), str(old_version_cached))) # Remove old repository keys and import new ones if _log is not None: _log.info('removing old repository keys') [rv, out, err] = run_command([ constants.CMD_RM, '-f', '/etc/apt/trustdb.gpg', '/etc/apt/trusted.gpg' ]) # XXX: more logging here [rv, out, err] = run_command([constants.CMD_APT_KEY, 'list' ]) # this recreates the gpg key store files if _log is not None: _log.info('importing repository keys') # XXX: more logging here [rv, out, err] = run_command([ constants.CMD_APT_KEY, 'add', constants.UPDATE_REPOSITORY_KEYS_FILE ]) if rv != 0: result = [9, 'failed to import repository keys'] raise UpdateError() # Cleanup old package list files, ignore errors _cleanup_old_package_lists() # Aptitude -f -y install _dpkg_configure_fixup() _aptitude_force_install() # In some practical situtions, e.g. in #734, the above command may fail # really badly (caused by broken package scripts), leading to a "dpkg # interrupted state". We re-run dpkg --configure -a to recover. _dpkg_configure_fixup() # Aptitude update if _log is not None: _log.info('updating package list') [rv, out, err] = _run_aptitude(['update']) if rv != 0: result = [4, 'package list update failed'] raise UpdateError() # Cleanup possibly confusing files in /tmp, should not be needed for f in os.listdir('/tmp'): m = v_name_re.match(f) if m is not None: [rv, out, err] = run_command( [constants.CMD_RM, '-f', os.path.join('/tmp', f)]) # XXX: sanity _dpkg_configure_fixup() # Test repository access if _log is not None: _log.info('testing repository access') [rv, out, err] = _run_aptitude(['download', 'vpnease'], cwd='/tmp') if rv != 0: result = [5, 'server not available'] raise UpdateError() # If testing for version info fails, it is ignored and it may generate an error # later when we check if the product version changed or not. for f in os.listdir('/tmp'): m = v_name_re.match(f) if m is not None: cmp_version = versioninfo.get_package_version_info( os.path.join('/tmp', f)) if cmp_version is not None and cmp_version == old_version: result = [ 0, 'update not run, version in repository matches current version' ] raise UpdateError() break # Aptitude clean if _log is not None: _log.info('cleaning up old package files') _run_aptitude(['clean']) # Aptitude upgrade # XXX: should this be dist-upgrade? dist-upgrade seems to be more forceful... _dpkg_configure_fixup() # XXX: sanity [rv1, out, err] = _run_aptitude(['upgrade']) # Aptitude install vpnease _dpkg_configure_fixup() # XXX: sanity [rv2, out, err ] = _run_aptitude(['install', 'vpnease' ]) # Ensure vpnease package is still installed # Reinstall dependencies of vpnease package (sanity) _dpkg_configure_fixup() # XXX: sanity try: deps = [] dep_re = re.compile('^Depends: (.*)$') # XXX: better logging here # XXX: check that package is installed correctly [rv, out, err] = run_command(['dpkg', '-s', 'vpnease']) for l in out.split('\n'): m = dep_re.match(l) if m is None: continue for d in m.groups()[0].split(','): ds = d.strip().split(' ') deps.append(ds[0]) _run_aptitude(['reinstall'] + deps) except: if _log is not None: _log.info('failed to reinstall vpnease depends') # Aptitude reinstall vpnease try: _run_aptitude(['reinstall', 'vpnease']) except: if _log is not None: _log.info('failed to reinstall vpnease depends') # Aptitude clean if _log is not None: _log.info('cleaning up downloaded package files') _run_aptitude(['clean']) # Get new version, determine whether update succeeded based on version & return values new_version, new_version_cached = versioninfo.get_version_info() if rv1 != 0 or rv2 != 0: result = [ 6, 'upgrade/install command failed with error value: %s/%s' % (str(rv1), str(rv2)) ] raise UpdateError() elif new_version_cached: result = [7, 'failed to read version information after update'] raise UpdateError() elif old_version == new_version: result = [ 8, 'product version unchanged after update, update failed' ] raise UpdateError() except UpdateError: pass except: result = [32, 'update failed with an exception'] # At this point, update was successful if result is None # Cleanup.. [rv, out, err] = run_command([constants.CMD_RM, '-f', '/tmp/vpnease_*.deb']) # Note: because timesync may be done later in update (after this script is run) # this timestamp may not be very accurate. def _write_marker_file(msg): # XXX: msg is not used for anything here ?? rv = helpers.write_datetime_marker_file( constants.UPDATE_RESULT_MARKER_FILE) if rv is None: raise Exception('failed to write update timestamp marker') # Handle errors if result is not None: if result[0] == 32: if _log is not None: _log.exception(result[1]) else: if _log is not None: _log.warning(result[1]) try: _write_marker_file('%s %s\nfailed\n%s' % (old_version, new_version, result[1])) except: if _log is not None: _log.exception( 'failed to write marker fail after failed update, ignoring' ) return result[0] # Update last update result marker file try: if _log is not None: _log.info('update success: %s -> %s' % (old_version, new_version)) _write_marker_file('%s %s\nsuccess\nUpdate success' % (old_version, new_version)) except: if _log is not None: _log.exception( 'failed to write marker file after successful update, ignoring' ) # Update last successful update timestamp, used by Web UI try: rv = helpers.write_datetime_marker_file( constants.LAST_SUCCESSFUL_UPDATE_MARKER_FILE) if rv is None: raise Exception( 'failed to write last successful update timestamp') except: if _log is not None: _log.exception( 'failed to write last successful update timestamp, ignoring' ) # Update done with success return 2
def get_apt_info(self, apt_sources_list): """Get apt information for an apt sources.list. Retrieves relevant information (with caching), and returns a tuple consisting of: latest version number, changelog. FIXME: has some trouble now with downgrade, maybe need to nuke /var/lib/apt/lists/vpnease* or something. Also, could we download lists to some temporary file instead? """ try: now = datetime.datetime.utcnow() # serve from cache? if self.cache.has_key(apt_sources_list): cinfo = self.cache[apt_sources_list] diff = now - cinfo.cachetime if (diff >= datetime.timedelta(0, 0, 0)) and ( diff <= datetime.timedelta(0, self.interval, 0)): _log.info('serving cached apt info for version %s' % cinfo.version) return cinfo.version, cinfo.changelog else: pass # no, fetch using apt and cache _log.info('fetching apt info, apt source:') _log.info(apt_sources_list) helpers.write_file(self.tmpsource, apt_sources_list, perms=0644) run_command(['aptitude'] + self.aptitude_options + ['update'], retval=runcommand.FAIL) run_command(['aptitude'] + self.aptitude_options + ['download', 'vpnease'], cwd='/tmp', retval=runcommand.FAIL) version = None for i in os.listdir('/tmp'): if self.package_name_re.match(i): version = versioninfo.get_package_version_info( os.path.join('/tmp', i)) changelog = versioninfo.get_package_changelog( os.path.join('/tmp', i)) run_command(['/bin/rm', '-f', os.path.join('/tmp', i)], retval=runcommand.FAIL) if version is None: raise Exception('vpnease package file not found') if changelog is None: raise Exception('vpnease package changelog not found') # create a new cache object cinfo = AptCacheInfo(now, version, changelog) self.cache[apt_sources_list] = cinfo # return fresh data _log.info('serving fresh apt info for version %s' % cinfo.version) return cinfo.version, cinfo.changelog except: _log.exception('failed to get package version info') raise raise Exception('should not be here')
def run_update(self): # Ensure no questions asked os.environ['DEBIAN_FRONTEND'] = 'noninteractive' # XXX: is this ok? # os.environ['DEBIAN_PRIORITY'] = 'critical' # os.environ['DEBCONF_NOWARNINGS'] = 'yes' result = None old_version = None old_version_cached = None new_version = None new_version_cached = None # Tolerate logger errors try: from codebay.common import logger _log = logger.get('l2tpserver.update.update') except: _log = None try: from codebay.l2tpserver import constants from codebay.l2tpserver import runcommand from codebay.l2tpserver import versioninfo from codebay.l2tpserver import helpers run_command = runcommand.run_command except: print 'failed to import helpers, cannot update' return 3 try: def _run_aptitude(options, cwd=None): """Run aptitude with automatic -q and -y options.""" # XXX: add some option to make aptitude return with error instead of asking to ignore trust violations => may not be possible # - if cannot be done, missing repository keys will result in update timeout (aptitude will block) cmd = [constants.CMD_APTITUDE, '-q', '-y'] + options if _log is not None: _log.info('running: %s' % ' '.join(cmd)) retval, stdout, stderr = run_command(cmd, cwd=cwd) if retval != 0: if _log is not None: _log.warning('aptitude return value: %d' % retval) _log.warning('stdout:\n%s' % stdout) _log.warning('stderr:\n%s' % stderr) return retval, stdout, stderr def _dpkg_configure_fixup(): """Attempt 'dpkg --configure -a' fixup.""" # XXX: the previous worry here is that something may block, and an update # might actually fix the situation anyway. try: # XXX: more logging, XXX: path use [rv, out, err] = run_command(['dpkg', '--force-depends', '--force-confmiss', '--force-confdef', '--force-confold', '--configure', '-a'], retval=runcommand.FAIL) except: if _log is not None: _log.exception('dpkg --configure -a failed, this is bad') def _cleanup_old_package_lists(): """Cleanup old package lists.""" try: if _log is not None: _log.info('cleaning up apt sources lists') # Remove all files in partial dir partial_dir = '/var/lib/apt/lists/partial' for f in os.listdir(partial_dir): fpath = os.path.join(partial_dir, f) if os.path.isfile(fpath): run_command(['/bin/rm', '-f', fpath]) # Remove all files except lock lists_dir = '/var/lib/apt/lists' for f in os.listdir(lists_dir): fpath = os.path.join(lists_dir, f) m = lockfile_name_re.match(f) if (os.path.isfile(fpath)) and (m is None): run_command(['/bin/rm', '-f', fpath]) except: if _log is not None: _log.exception('failed to remove package list files') def _aptitude_force_install(): """Attempt aptitude install which may clear up some (but not nearly all) error situations.""" try: # XXX: more logging here # XXX: add -q (quiet) option? run_command([constants.CMD_APTITUDE, '-f', '-y', 'install'], retval=runcommand.FAIL) except: if _log is not None: _log.exception('apt-get cleanup failed, ignoring') # Figure out version v_name_re = re.compile('vpnease_.*\.deb') lockfile_name_re = re.compile('^lock$') old_version, old_version_cached = versioninfo.get_version_info() if _log is not None: _log.info('starting update, current_version: %s (cached: %s)' % (str(old_version), str(old_version_cached))) # Remove old repository keys and import new ones if _log is not None: _log.info('removing old repository keys') [rv, out, err] = run_command([constants.CMD_RM, '-f', '/etc/apt/trustdb.gpg', '/etc/apt/trusted.gpg']) # XXX: more logging here [rv, out, err] = run_command([constants.CMD_APT_KEY, 'list']) # this recreates the gpg key store files if _log is not None: _log.info('importing repository keys') # XXX: more logging here [rv, out, err] = run_command([constants.CMD_APT_KEY, 'add', constants.UPDATE_REPOSITORY_KEYS_FILE]) if rv != 0: result = [9, 'failed to import repository keys'] raise UpdateError() # Cleanup old package list files, ignore errors _cleanup_old_package_lists() # Aptitude -f -y install _dpkg_configure_fixup() _aptitude_force_install() # In some practical situtions, e.g. in #734, the above command may fail # really badly (caused by broken package scripts), leading to a "dpkg # interrupted state". We re-run dpkg --configure -a to recover. _dpkg_configure_fixup() # Aptitude update if _log is not None: _log.info('updating package list') [rv, out, err] = _run_aptitude(['update']) if rv != 0: result = [4, 'package list update failed'] raise UpdateError() # Cleanup possibly confusing files in /tmp, should not be needed for f in os.listdir('/tmp'): m = v_name_re.match(f) if m is not None: [rv, out, err] = run_command([constants.CMD_RM, '-f', os.path.join('/tmp', f)]) # XXX: sanity _dpkg_configure_fixup() # Test repository access if _log is not None: _log.info('testing repository access') [rv, out, err] = _run_aptitude(['download', 'vpnease'], cwd='/tmp') if rv != 0: result = [5, 'server not available'] raise UpdateError() # If testing for version info fails, it is ignored and it may generate an error # later when we check if the product version changed or not. for f in os.listdir('/tmp'): m = v_name_re.match(f) if m is not None: cmp_version = versioninfo.get_package_version_info(os.path.join('/tmp', f)) if cmp_version is not None and cmp_version == old_version: result = [0, 'update not run, version in repository matches current version'] raise UpdateError() break # Aptitude clean if _log is not None: _log.info('cleaning up old package files') _run_aptitude(['clean']) # Aptitude upgrade # XXX: should this be dist-upgrade? dist-upgrade seems to be more forceful... _dpkg_configure_fixup() # XXX: sanity [rv1, out, err] = _run_aptitude(['upgrade']) # Aptitude install vpnease _dpkg_configure_fixup() # XXX: sanity [rv2, out, err] = _run_aptitude(['install', 'vpnease']) # Ensure vpnease package is still installed # Reinstall dependencies of vpnease package (sanity) _dpkg_configure_fixup() # XXX: sanity try: deps = [] dep_re = re.compile('^Depends: (.*)$') # XXX: better logging here # XXX: check that package is installed correctly [rv, out, err] = run_command(['dpkg', '-s', 'vpnease']) for l in out.split('\n'): m = dep_re.match(l) if m is None: continue for d in m.groups()[0].split(','): ds = d.strip().split(' ') deps.append(ds[0]) _run_aptitude(['reinstall'] + deps) except: if _log is not None: _log.info('failed to reinstall vpnease depends') # Aptitude reinstall vpnease try: _run_aptitude(['reinstall', 'vpnease']) except: if _log is not None: _log.info('failed to reinstall vpnease depends') # Aptitude clean if _log is not None: _log.info('cleaning up downloaded package files') _run_aptitude(['clean']) # Get new version, determine whether update succeeded based on version & return values new_version, new_version_cached = versioninfo.get_version_info() if rv1 != 0 or rv2 != 0: result = [6, 'upgrade/install command failed with error value: %s/%s' % (str(rv1), str(rv2))] raise UpdateError() elif new_version_cached: result = [7, 'failed to read version information after update'] raise UpdateError() elif old_version == new_version: result = [8, 'product version unchanged after update, update failed'] raise UpdateError() except UpdateError: pass except: result = [32, 'update failed with an exception'] # At this point, update was successful if result is None # Cleanup.. [rv, out, err] = run_command([constants.CMD_RM, '-f', '/tmp/vpnease_*.deb']) # Note: because timesync may be done later in update (after this script is run) # this timestamp may not be very accurate. def _write_marker_file(msg): # XXX: msg is not used for anything here ?? rv = helpers.write_datetime_marker_file(constants.UPDATE_RESULT_MARKER_FILE) if rv is None: raise Exception('failed to write update timestamp marker') # Handle errors if result is not None: if result[0] == 32: if _log is not None: _log.exception(result[1]) else: if _log is not None: _log.warning(result[1]) try: _write_marker_file('%s %s\nfailed\n%s' % (old_version, new_version, result[1])) except: if _log is not None: _log.exception('failed to write marker fail after failed update, ignoring') return result[0] # Update last update result marker file try: if _log is not None: _log.info('update success: %s -> %s' % (old_version, new_version)) _write_marker_file('%s %s\nsuccess\nUpdate success' % (old_version, new_version)) except: if _log is not None: _log.exception('failed to write marker file after successful update, ignoring') # Update last successful update timestamp, used by Web UI try: rv = helpers.write_datetime_marker_file(constants.LAST_SUCCESSFUL_UPDATE_MARKER_FILE) if rv is None: raise Exception('failed to write last successful update timestamp') except: if _log is not None: _log.exception('failed to write last successful update timestamp, ignoring') # Update done with success return 2