def _parse_demo_license(self, license_key): re_keyval = re.compile(r'^(.*?)=(.*?)$') grant_time = None remote_address = None remote_port = None installation_uuid = None f = None try: try: fname = str(os.path.join(msconstants.DEMO_LICENSE_DIRECTORY, license_key)) f = open(fname, 'rb') for line in f.readlines(): line = line.strip() m = re_keyval.match(line) if m is not None: g = m.groups() if len(g) == 2: key, value = g[0], g[1] if key == 'grant-time': grant_time = datatypes.parse_datetime_from_iso8601_subset(value) elif key == 'remote-address': remote_address = datatypes.IPv4Address.fromString(value) elif key == 'remote-port': remote_port = int(value) elif key == 'installation-uuid': installation_uuid = str(value) else: _log.warning('skipping demo license key-value pair: %s=%s' % (key,value)) except IOError: # eat error pass except: _log.exception('failed in parsing demo license file for license key %s' % license_key) raise finally: if f is not None: f.close() if (grant_time is not None) and (remote_address is not None) and (remote_port is not None) and (installation_uuid is not None): return DemoLicense(license_key, grant_time, remote_address, remote_port, installation_uuid, None) else: return None
def read_datetime_marker_file(fname): """Reads a timestamp from a markerfile. Returns the datetime object parsed from the timestamp in markerfile if found and valid, None otherwise. """ timestamp = None f = None try: f = open(fname, 'rb') timestamp = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip()) except: pass if f is not None: f.close() return timestamp
def read_datetime_marker_file(fname): """Reads a timestamp from a markerfile. Returns the datetime object parsed from the timestamp in markerfile if found and valid, None otherwise. """ timestamp = None f = None try: f = open(fname, 'rb') timestamp = datatypes.parse_datetime_from_iso8601_subset( f.readline().strip()) except: pass if f is not None: f.close() return timestamp
def _reboot(): """Reboot system after web UI health check failure. Only reboot if the boot timestamp is old enough to ensure that (a) we don't panic when the system is just starting up, and (b) if there is a reboot loop, slow it down to something manageable. Resets the failure count even when reboot is not done so that later errors would not cause immediate reboot after. """ try: if os.path.exists(constants.BOOT_TIMESTAMP_FILE): f = open(constants.BOOT_TIMESTAMP_FILE, 'r') boottime = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip()) _log.info('_reboot: last boot timestamp: %s' % boottime) currenttime = datetime.datetime.utcnow() _log.info('_reboot: current time %s' % currenttime) runtime = currenttime - boottime _log.info('_reboot: running time: %s (limit: %s)' % (runtime, constants.CRON_BOOTTIME_FAILURE_WAIT)) if runtime < constants.CRON_BOOTTIME_FAILURE_WAIT and runtime > constants.ZERO_TIMEDELTA: _log.info('_reboot: too soon to reboot despite UI health check failures') _reset_failure_count() return except: _log.exception('_reboot: failed to check boot timestamp, rebooting anyway') _log.warning('_reboot: rebooting due to UI health check failures') _reset_failure_count() # Launch a notify in the background: may or may not work try: run_command(['%s watchdognotify 1>/dev/null 2>/dev/null &' % constants.CMD_L2TPGW_CRON], shell=True) except: _log.exception('failed to launch notify in the background') # Simulate normal update behaviour and force fsck on next # boot try: if os.path.exists(constants.UPDATE_FORCE_MARKER_FILE): os.unlink(constants.UPDATE_FORCE_MARKER_FILE) if os.path.exists(constants.UPDATE_SKIP_MARKER_FILE): os.unlink(constants.UPDATE_SKIP_MARKER_FILE) if os.path.exists(constants.FASTBOOT_MARKER_FILE): os.unlink(constants.FASTBOOT_MARKER_FILE) helpers.write_datetime_marker_file(constants.FORCE_FSCK_MARKER_FILE) except: _log.exception('marker file cleanup failed, but we reboot anyway') # Give some time for notify to show try: _log.info('sleeping %s seconds before rebooting' % constants.CRON_WATCHDOG_REBOOT_DELAY) time.sleep(constants.CRON_WATCHDOG_REBOOT_DELAY) except: _log.exception('failed to sleep in cron watchdog action, ignoring') # Flush syslog to disk try: _log.info('flushing syslog to disk') pid = int(open(constants.SYSLOG_PIDFILE, 'rb').read().strip()) # do not care of fd leak here os.kill(pid, signal.SIGHUP) time.sleep(1) except: _log.exception('failed to flush syslog, ignoring') # Sync disk and force reboot: this avoids possible start/stop # system script blocking try: run_command(constants.CMD_SYNC) run_command(constants.CMD_SYNC) run_command(constants.CMD_SYNC) except: _log.exception('failed to do filesystem sync, ignoring') run_command([constants.CMD_REBOOT, '-f', '-d'])
def _check_ui_health(): """Check UI process health and reboot if required. """ def _reset_failure_count(): try: run_command([constants.CMD_RM, '-f', constants.CRON_WEBUI_FAILURE_COUNT_FILE]) except: _log.exception('_reset_failure_count failed') def _reboot(): """Reboot system after web UI health check failure. Only reboot if the boot timestamp is old enough to ensure that (a) we don't panic when the system is just starting up, and (b) if there is a reboot loop, slow it down to something manageable. Resets the failure count even when reboot is not done so that later errors would not cause immediate reboot after. """ try: if os.path.exists(constants.BOOT_TIMESTAMP_FILE): f = open(constants.BOOT_TIMESTAMP_FILE, 'r') boottime = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip()) _log.info('_reboot: last boot timestamp: %s' % boottime) currenttime = datetime.datetime.utcnow() _log.info('_reboot: current time %s' % currenttime) runtime = currenttime - boottime _log.info('_reboot: running time: %s (limit: %s)' % (runtime, constants.CRON_BOOTTIME_FAILURE_WAIT)) if runtime < constants.CRON_BOOTTIME_FAILURE_WAIT and runtime > constants.ZERO_TIMEDELTA: _log.info('_reboot: too soon to reboot despite UI health check failures') _reset_failure_count() return except: _log.exception('_reboot: failed to check boot timestamp, rebooting anyway') _log.warning('_reboot: rebooting due to UI health check failures') _reset_failure_count() # Launch a notify in the background: may or may not work try: run_command(['%s watchdognotify 1>/dev/null 2>/dev/null &' % constants.CMD_L2TPGW_CRON], shell=True) except: _log.exception('failed to launch notify in the background') # Simulate normal update behaviour and force fsck on next # boot try: if os.path.exists(constants.UPDATE_FORCE_MARKER_FILE): os.unlink(constants.UPDATE_FORCE_MARKER_FILE) if os.path.exists(constants.UPDATE_SKIP_MARKER_FILE): os.unlink(constants.UPDATE_SKIP_MARKER_FILE) if os.path.exists(constants.FASTBOOT_MARKER_FILE): os.unlink(constants.FASTBOOT_MARKER_FILE) helpers.write_datetime_marker_file(constants.FORCE_FSCK_MARKER_FILE) except: _log.exception('marker file cleanup failed, but we reboot anyway') # Give some time for notify to show try: _log.info('sleeping %s seconds before rebooting' % constants.CRON_WATCHDOG_REBOOT_DELAY) time.sleep(constants.CRON_WATCHDOG_REBOOT_DELAY) except: _log.exception('failed to sleep in cron watchdog action, ignoring') # Flush syslog to disk try: _log.info('flushing syslog to disk') pid = int(open(constants.SYSLOG_PIDFILE, 'rb').read().strip()) # do not care of fd leak here os.kill(pid, signal.SIGHUP) time.sleep(1) except: _log.exception('failed to flush syslog, ignoring') # Sync disk and force reboot: this avoids possible start/stop # system script blocking try: run_command(constants.CMD_SYNC) run_command(constants.CMD_SYNC) run_command(constants.CMD_SYNC) except: _log.exception('failed to do filesystem sync, ignoring') run_command([constants.CMD_REBOOT, '-f', '-d']) def _failure(reason='unknown reason'): """Update failure count and reboot if limit is reached.""" _log.warning('web UI health check failed: %s' % reason) try: if not os.path.exists(constants.CRON_WEBUI_FAILURE_COUNT_FILE): count = 1 f = open(constants.CRON_WEBUI_FAILURE_COUNT_FILE, 'w') f.write('%s\n' % str(count)) f.close() _log.info('web UI health check failure count now: %s' % count) else: f = open(constants.CRON_WEBUI_FAILURE_COUNT_FILE, 'r') count = int(f.readline().strip()) f.close() count += 1 _log.info('web UI health check failure count now: %s' % count) f = open(constants.CRON_WEBUI_FAILURE_COUNT_FILE, 'w') f.write('%s\n' % str(count)) f.close() if count > 3 or count < 1: _reboot() except: _log.warning('failed to handle health check failure properly, rebooting') _reboot() # skip checks if Live CD if os.path.exists(constants.LIVECD_MARKER_FILE): return # web UI should always be running if not os.path.exists(constants.WEBUI_PIDFILE): _failure('UI pidfile missing') return # check that web UI process is alive try: f = open(constants.WEBUI_PIDFILE, 'r') pid = int(f.readline().strip()) f.close() if os.system(constants.CMD_KILL + " -0 " + str(pid)) != 0: _failure('UI process missing') return except: _failure('Invalid UI pidfile') return # check that web UI watchdog is active enough if not os.path.exists(constants.WEBUI_WATCHDOG_LAST_UPDATED_FILE): _failure('UI watchdog markerfile missing') return f = open(constants.WEBUI_WATCHDOG_LAST_UPDATED_FILE, 'r') checktime = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip()) _log.debug('last webui watchdog timestamp: %s' % checktime) currenttime = datetime.datetime.utcnow() _log.debug('current time: %s' % currenttime) age = currenttime - checktime _log.debug('webui watchdog timestamp age: %s (limit: %s)' % (age, constants.CRON_WEBUI_WATCHDOG_TIMEOUT)) if age > constants.CRON_WEBUI_WATCHDOG_TIMEOUT or age <= constants.ZERO_TIMEDELTA: _failure('UI watchdog markerfile too old.') return # health check ok _reset_failure_count() _log.debug('UI health check passed')