def _parse_demo_license(self, license_key):
re_keyval = re.compile(r'^(.*?)=(.*?)$')
grant_time = None
remote_address = None
remote_port = None
installation_uuid = None
f = None
try:
try:
fname = str(os.path.join(msconstants.DEMO_LICENSE_DIRECTORY, license_key))
f = open(fname, 'rb')
for line in f.readlines():
line = line.strip()
m = re_keyval.match(line)
if m is not None:
g = m.groups()
if len(g) == 2:
key, value = g[0], g[1]
if key == 'grant-time':
grant_time = datatypes.parse_datetime_from_iso8601_subset(value)
elif key == 'remote-address':
remote_address = datatypes.IPv4Address.fromString(value)
elif key == 'remote-port':
remote_port = int(value)
elif key == 'installation-uuid':
installation_uuid = str(value)
else:
_log.warning('skipping demo license key-value pair: %s=%s' % (key,value))
except IOError:
# eat error
pass
except:
_log.exception('failed in parsing demo license file for license key %s' % license_key)
raise
finally:
if f is not None:
f.close()
if (grant_time is not None) and (remote_address is not None) and (remote_port is not None) and (installation_uuid is not None):
return DemoLicense(license_key, grant_time, remote_address, remote_port, installation_uuid, None)
else:
return None
def read_datetime_marker_file(fname):
"""Reads a timestamp from a markerfile.
Returns the datetime object parsed from the timestamp in
markerfile if found and valid, None otherwise.
"""
timestamp = None
f = None
try:
f = open(fname, 'rb')
timestamp = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip())
except:
pass
if f is not None:
f.close()
return timestamp
def read_datetime_marker_file(fname):
"""Reads a timestamp from a markerfile.
Returns the datetime object parsed from the timestamp in
markerfile if found and valid, None otherwise.
"""
timestamp = None
f = None
try:
f = open(fname, 'rb')
timestamp = datatypes.parse_datetime_from_iso8601_subset(
f.readline().strip())
except:
pass
if f is not None:
f.close()
return timestamp
def _reboot():
"""Reboot system after web UI health check failure.
Only reboot if the boot timestamp is old enough to ensure that (a) we don't panic
when the system is just starting up, and (b) if there is a reboot loop, slow it
down to something manageable.
Resets the failure count even when reboot is not done so that later errors would
not cause immediate reboot after.
"""
try:
if os.path.exists(constants.BOOT_TIMESTAMP_FILE):
f = open(constants.BOOT_TIMESTAMP_FILE, 'r')
boottime = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip())
_log.info('_reboot: last boot timestamp: %s' % boottime)
currenttime = datetime.datetime.utcnow()
_log.info('_reboot: current time %s' % currenttime)
runtime = currenttime - boottime
_log.info('_reboot: running time: %s (limit: %s)' % (runtime, constants.CRON_BOOTTIME_FAILURE_WAIT))
if runtime < constants.CRON_BOOTTIME_FAILURE_WAIT and runtime > constants.ZERO_TIMEDELTA:
_log.info('_reboot: too soon to reboot despite UI health check failures')
_reset_failure_count()
return
except:
_log.exception('_reboot: failed to check boot timestamp, rebooting anyway')
_log.warning('_reboot: rebooting due to UI health check failures')
_reset_failure_count()
# Launch a notify in the background: may or may not work
try:
run_command(['%s watchdognotify 1>/dev/null 2>/dev/null &' % constants.CMD_L2TPGW_CRON], shell=True)
except:
_log.exception('failed to launch notify in the background')
# Simulate normal update behaviour and force fsck on next
# boot
try:
if os.path.exists(constants.UPDATE_FORCE_MARKER_FILE):
os.unlink(constants.UPDATE_FORCE_MARKER_FILE)
if os.path.exists(constants.UPDATE_SKIP_MARKER_FILE):
os.unlink(constants.UPDATE_SKIP_MARKER_FILE)
if os.path.exists(constants.FASTBOOT_MARKER_FILE):
os.unlink(constants.FASTBOOT_MARKER_FILE)
helpers.write_datetime_marker_file(constants.FORCE_FSCK_MARKER_FILE)
except:
_log.exception('marker file cleanup failed, but we reboot anyway')
# Give some time for notify to show
try:
_log.info('sleeping %s seconds before rebooting' % constants.CRON_WATCHDOG_REBOOT_DELAY)
time.sleep(constants.CRON_WATCHDOG_REBOOT_DELAY)
except:
_log.exception('failed to sleep in cron watchdog action, ignoring')
# Flush syslog to disk
try:
_log.info('flushing syslog to disk')
pid = int(open(constants.SYSLOG_PIDFILE, 'rb').read().strip()) # do not care of fd leak here
os.kill(pid, signal.SIGHUP)
time.sleep(1)
except:
_log.exception('failed to flush syslog, ignoring')
# Sync disk and force reboot: this avoids possible start/stop
# system script blocking
try:
run_command(constants.CMD_SYNC)
run_command(constants.CMD_SYNC)
run_command(constants.CMD_SYNC)
except:
_log.exception('failed to do filesystem sync, ignoring')
run_command([constants.CMD_REBOOT, '-f', '-d'])
def _check_ui_health():
"""Check UI process health and reboot if required.
"""
def _reset_failure_count():
try:
run_command([constants.CMD_RM, '-f', constants.CRON_WEBUI_FAILURE_COUNT_FILE])
except:
_log.exception('_reset_failure_count failed')
def _reboot():
"""Reboot system after web UI health check failure.
Only reboot if the boot timestamp is old enough to ensure that (a) we don't panic
when the system is just starting up, and (b) if there is a reboot loop, slow it
down to something manageable.
Resets the failure count even when reboot is not done so that later errors would
not cause immediate reboot after.
"""
try:
if os.path.exists(constants.BOOT_TIMESTAMP_FILE):
f = open(constants.BOOT_TIMESTAMP_FILE, 'r')
boottime = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip())
_log.info('_reboot: last boot timestamp: %s' % boottime)
currenttime = datetime.datetime.utcnow()
_log.info('_reboot: current time %s' % currenttime)
runtime = currenttime - boottime
_log.info('_reboot: running time: %s (limit: %s)' % (runtime, constants.CRON_BOOTTIME_FAILURE_WAIT))
if runtime < constants.CRON_BOOTTIME_FAILURE_WAIT and runtime > constants.ZERO_TIMEDELTA:
_log.info('_reboot: too soon to reboot despite UI health check failures')
_reset_failure_count()
return
except:
_log.exception('_reboot: failed to check boot timestamp, rebooting anyway')
_log.warning('_reboot: rebooting due to UI health check failures')
_reset_failure_count()
# Launch a notify in the background: may or may not work
try:
run_command(['%s watchdognotify 1>/dev/null 2>/dev/null &' % constants.CMD_L2TPGW_CRON], shell=True)
except:
_log.exception('failed to launch notify in the background')
# Simulate normal update behaviour and force fsck on next
# boot
try:
if os.path.exists(constants.UPDATE_FORCE_MARKER_FILE):
os.unlink(constants.UPDATE_FORCE_MARKER_FILE)
if os.path.exists(constants.UPDATE_SKIP_MARKER_FILE):
os.unlink(constants.UPDATE_SKIP_MARKER_FILE)
if os.path.exists(constants.FASTBOOT_MARKER_FILE):
os.unlink(constants.FASTBOOT_MARKER_FILE)
helpers.write_datetime_marker_file(constants.FORCE_FSCK_MARKER_FILE)
except:
_log.exception('marker file cleanup failed, but we reboot anyway')
# Give some time for notify to show
try:
_log.info('sleeping %s seconds before rebooting' % constants.CRON_WATCHDOG_REBOOT_DELAY)
time.sleep(constants.CRON_WATCHDOG_REBOOT_DELAY)
except:
_log.exception('failed to sleep in cron watchdog action, ignoring')
# Flush syslog to disk
try:
_log.info('flushing syslog to disk')
pid = int(open(constants.SYSLOG_PIDFILE, 'rb').read().strip()) # do not care of fd leak here
os.kill(pid, signal.SIGHUP)
time.sleep(1)
except:
_log.exception('failed to flush syslog, ignoring')
# Sync disk and force reboot: this avoids possible start/stop
# system script blocking
try:
run_command(constants.CMD_SYNC)
run_command(constants.CMD_SYNC)
run_command(constants.CMD_SYNC)
except:
_log.exception('failed to do filesystem sync, ignoring')
run_command([constants.CMD_REBOOT, '-f', '-d'])
def _failure(reason='unknown reason'):
"""Update failure count and reboot if limit is reached."""
_log.warning('web UI health check failed: %s' % reason)
try:
if not os.path.exists(constants.CRON_WEBUI_FAILURE_COUNT_FILE):
count = 1
f = open(constants.CRON_WEBUI_FAILURE_COUNT_FILE, 'w')
f.write('%s\n' % str(count))
f.close()
_log.info('web UI health check failure count now: %s' % count)
else:
f = open(constants.CRON_WEBUI_FAILURE_COUNT_FILE, 'r')
count = int(f.readline().strip())
f.close()
count += 1
_log.info('web UI health check failure count now: %s' % count)
f = open(constants.CRON_WEBUI_FAILURE_COUNT_FILE, 'w')
f.write('%s\n' % str(count))
f.close()
if count > 3 or count < 1:
_reboot()
except:
_log.warning('failed to handle health check failure properly, rebooting')
_reboot()
# skip checks if Live CD
if os.path.exists(constants.LIVECD_MARKER_FILE):
return
# web UI should always be running
if not os.path.exists(constants.WEBUI_PIDFILE):
_failure('UI pidfile missing')
return
# check that web UI process is alive
try:
f = open(constants.WEBUI_PIDFILE, 'r')
pid = int(f.readline().strip())
f.close()
if os.system(constants.CMD_KILL + " -0 " + str(pid)) != 0:
_failure('UI process missing')
return
except:
_failure('Invalid UI pidfile')
return
# check that web UI watchdog is active enough
if not os.path.exists(constants.WEBUI_WATCHDOG_LAST_UPDATED_FILE):
_failure('UI watchdog markerfile missing')
return
f = open(constants.WEBUI_WATCHDOG_LAST_UPDATED_FILE, 'r')
checktime = datatypes.parse_datetime_from_iso8601_subset(f.readline().strip())
_log.debug('last webui watchdog timestamp: %s' % checktime)
currenttime = datetime.datetime.utcnow()
_log.debug('current time: %s' % currenttime)
age = currenttime - checktime
_log.debug('webui watchdog timestamp age: %s (limit: %s)' % (age, constants.CRON_WEBUI_WATCHDOG_TIMEOUT))
if age > constants.CRON_WEBUI_WATCHDOG_TIMEOUT or age <= constants.ZERO_TIMEDELTA:
_failure('UI watchdog markerfile too old.')
return
# health check ok
_reset_failure_count()
_log.debug('UI health check passed')
