def _do_timesync(self, utc_time):
# XXX: if time step is too large, we need to take some measures here to prevent twisted problems
# (Twisted does not handle time jumps correctly; if time goes backwards, timers become frozen).
# One relatively benign fix here would be to reschedule all critical timers, such as the watchdog.
# A more brute force fix would be to restart the web UI.
try:
full_sync = timesync.update_system_time(
utc_time,
cap_backwards=constants.WEBUI_TIMESYNC_CAP_BACKWARDS,
cap_forwards=constants.WEBUI_TIMESYNC_CAP_FORWARDS,
)
if full_sync:
helpers.write_datetime_marker_file(constants.WEBUI_LAST_TIMESYNC_FILE)
else:
# probably capped, don't write marker because it would enable RRD with bogus system time
_log.info("timesync not full (probably capped), not writing webui timesync file")
# notify if time difference is still too large (jump was capped too heavily)
now = datetime.datetime.utcnow()
timediff = utc_time - now
if timediff < datetime.timedelta(0, 0, 0):
timediff = -timediff
if (timediff > constants.WEBUI_TIMESYNC_NOTIFY_LIMIT) and (not self._timesync_notify_shown):
try:
from codebay.l2tpserver import gnomehelpers
gnomehelpers.show_notification(
constants.WEBUI_TIMESYNC_NOTIFY_TITLE,
constants.WEBUI_TIMESYNC_NOTIFY_TEXT,
timeout=constants.WEBUI_TIMESYNC_NOTIFY_TIMEOUT,
critical=False,
)
# NB: it is important to do this after show_notification(); if, for instance,
# boot is in progress, the initial notification will fail and a notification
# will only be shown on reidentify. Not ideal, but at least the notification
# will not be completely hidden.
self._timesync_notify_shown = True
except:
_log.exception("_do_timesync(): failed to show notify of time jump")
except:
_log.exception("time sync failed")
def run(self):
# remove default database just in case
from codebay.l2tpserver import db
db.remove_database()
# log informative stuff into log first
try:
self.log_update_info()
except:
_log.exception('log_update_info() failed, ignoring')
# prepare; may raise exceptions which propagate directly to caller
try:
self.prepare()
except:
_log.exception('prepare() failed')
raise
# run twisted, exits when protocol run complete
try:
self.run_twisted()
except:
_log.exception('run_twisted() failed')
raise
# do timesync last, to avoid disrupting timers
if self.server_utctime_received_at is None:
_log.info(
'management server connection time not available, ignoring timesync.'
)
else:
if self.do_timesync:
try:
# adjust server supplied time with local difference
# this is not very nice, but we want to do timesync last
now = datetime.datetime.utcnow()
diff = now - self.server_utctime_received_at
if diff > datetime.timedelta(0, 60 * 60, 0): # 1 hour
# Something's badly wrong, system time apparently jumped.
# This can happen e.g. if system time is updated by distro
# scripts when runner restarts distro networking.
#
# If this happens, we just zero the diff: this causes inaccuracy
# in time sync (<1 min) but is better than jumping arbitrarily.
_log.warning(
'time jump when attempting to sync time, diff is %s; zeroing'
% diff)
diff = datetime.timedelta(0, 0, 0)
dt = self.server_utctime + diff
_log.debug(
'doing timesync: server time before adjustment: %s, server time after adjustment: %s, received-at: %s, time-now:%s, diff: %s'
% (self.server_utctime, dt,
self.server_utctime_received_at, now, diff))
# update time, don't cap (allow arbitrary time jumps)
timesync.update_system_time(dt,
cap_backwards=None,
cap_forwards=None)
helpers.write_datetime_marker_file(
constants.UPDATE_TIMESYNC_TIMESTAMP_FILE)
except:
_log.exception(
'timesync with management server failed, ignoring.')
# return value handling
#
# Overall the deferred chain started by runner and management connection
# ends up in success or failure. We may or may not get a process exit
# code, depending on what is executed. Finally, if a timeout occurs,
# this error is flagged specially in self.update_failed.
if isinstance(self.update_exit_code, (int, long)):
if self.update_exit_code == 0:
# update run, did not try update => signaled as error (but supported case)
raise UpdateNotDoneError(
'policy requires update, but update cannot be performed')
else:
if self.update_exit_code == 2:
# update run, update succeeded => signaled as no exception, success case
raise UpdateDone(
'policy requires update and update was successful')
else:
# update run, update failed or unknown error
raise UpdateFailedError(
'update script failed with exit code: %s' %
self.update_exit_code)
else:
# update was not executed [or exit code is corrupted (should not happen)]
# typically we come here after a connection timeout when we try to update and/or timesync
if self.update_failed: # from global errback
# update not run, but failure (probably timeout)
raise UpdateUnknownError('unknown error (errback)')
else:
# policy does not require an update, no action was taken, success
raise UpdateNotRequired('update not required by policy')
raise UpdateUnknownError('should not be here')
def run(self):
# remove default database just in case
from codebay.l2tpserver import db
db.remove_database()
# log informative stuff into log first
try:
self.log_update_info()
except:
_log.exception('log_update_info() failed, ignoring')
# prepare; may raise exceptions which propagate directly to caller
try:
self.prepare()
except:
_log.exception('prepare() failed')
raise
# run twisted, exits when protocol run complete
try:
self.run_twisted()
except:
_log.exception('run_twisted() failed')
raise
# do timesync last, to avoid disrupting timers
if self.server_utctime_received_at is None:
_log.info('management server connection time not available, ignoring timesync.')
else:
if self.do_timesync:
try:
# adjust server supplied time with local difference
# this is not very nice, but we want to do timesync last
now = datetime.datetime.utcnow()
diff = now - self.server_utctime_received_at
if diff > datetime.timedelta(0, 60*60, 0): # 1 hour
# Something's badly wrong, system time apparently jumped.
# This can happen e.g. if system time is updated by distro
# scripts when runner restarts distro networking.
#
# If this happens, we just zero the diff: this causes inaccuracy
# in time sync (<1 min) but is better than jumping arbitrarily.
_log.warning('time jump when attempting to sync time, diff is %s; zeroing' % diff)
diff = datetime.timedelta(0, 0, 0)
dt = self.server_utctime + diff
_log.debug('doing timesync: server time before adjustment: %s, server time after adjustment: %s, received-at: %s, time-now:%s, diff: %s' % (self.server_utctime, dt, self.server_utctime_received_at, now, diff))
# update time, don't cap (allow arbitrary time jumps)
timesync.update_system_time(dt, cap_backwards=None, cap_forwards=None)
helpers.write_datetime_marker_file(constants.UPDATE_TIMESYNC_TIMESTAMP_FILE)
except:
_log.exception('timesync with management server failed, ignoring.')
# return value handling
#
# Overall the deferred chain started by runner and management connection
# ends up in success or failure. We may or may not get a process exit
# code, depending on what is executed. Finally, if a timeout occurs,
# this error is flagged specially in self.update_failed.
if isinstance(self.update_exit_code, (int, long)):
if self.update_exit_code == 0:
# update run, did not try update => signaled as error (but supported case)
raise UpdateNotDoneError('policy requires update, but update cannot be performed')
else:
if self.update_exit_code == 2:
# update run, update succeeded => signaled as no exception, success case
raise UpdateDone('policy requires update and update was successful')
else:
# update run, update failed or unknown error
raise UpdateFailedError('update script failed with exit code: %s' % self.update_exit_code)
else:
# update was not executed [or exit code is corrupted (should not happen)]
# typically we come here after a connection timeout when we try to update and/or timesync
if self.update_failed: # from global errback
# update not run, but failure (probably timeout)
raise UpdateUnknownError('unknown error (errback)')
else:
# policy does not require an update, no action was taken, success
raise UpdateNotRequired('update not required by policy')
raise UpdateUnknownError('should not be here')
