def _process_sm_exc(self, db_session, exc, msg, new_state=None):
"""
Process a state machine exception. For putting as much code as
possible on a common call path.
"""
delay_factor = 1 + random()
delay_period = timedelta(minutes=delay_factor *
float(settings['master_hold_timeout']))
old_state = self.state
if new_state:
self.state = new_state
log_msg = str(exc)
log_info(log_msg)
self.retry_msg = log_msg
create_event(ServerSMConfigure,
db_session=db_session,
sm_id=self.id_,
server_id=self.id_,
delay=delay_period,
server_name=self.name)
state_str = ''
if old_state != self.state:
state_str = ("old state %s, new state %s - " %
(old_state, self.state))
return (RCODE_OK, "Server '%s': %s%s" % (self.name, state_str, msg))
def _ready_all_reconfig(self, event):
"""
Process an all reconfig
"""
# Update server address info
recalc_machine_dns_server_info(event.db_session)
# Update master server configuration
rcode, msg = self._master_rndc(event, 'reconfig')
if rcode != RCODE_OK:
return (rcode, msg)
# Issue all reconfig event to all servers
delay_time = timedelta(
seconds=float(settings['master_rndc_settle_delay']))
db_session = event.db_session
ServerGroup = sql_types['ServerGroup']
for sg in db_session.query(ServerGroup):
try:
sg.write_config(db_session, ServerConfigFileError)
except ServerConfigFileError as exc:
log_error(str(exc))
continue
for server_sm in sg.servers:
if server_sm.is_disabled():
continue
create_event(sql_events['ServerSMConfigChange'],
db_session=event.db_session,
sm_id=server_sm.id_, server_id=server_sm.id_,
delay=delay_time,
server_name=server_sm.name)
self._hold_enter(db_session)
return (RCODE_OK,
"Master named reconfig done and reconfig queued for all SGs")
def reset_master_sm(db_session):
"""
Reset the Configuration state machine
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMReset, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_)
def reconfig_all(db_session):
"""
Reconfigure all DNS servers - helper
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMAllReconfig, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_)
def _ready_all_reconfig(self, event):
"""
Process an all reconfig
"""
# Update server address info
recalc_machine_dns_server_info(event.db_session)
# Update master server configuration
rcode, msg = self._master_rndc(event, 'reconfig')
if rcode != RCODE_OK:
return (rcode, msg)
# Issue all reconfig event to all servers
delay_time = timedelta(
seconds=float(settings['master_rndc_settle_delay']))
db_session = event.db_session
ServerGroup = sql_types['ServerGroup']
for sg in db_session.query(ServerGroup):
try:
sg.write_config(db_session, ServerConfigFileError)
except ServerConfigFileError as exc:
log_error(str(exc))
continue
for server_sm in sg.servers:
if server_sm.is_disabled():
continue
create_event(sql_events['ServerSMConfigChange'],
db_session=event.db_session,
sm_id=server_sm.id_,
server_id=server_sm.id_,
delay=delay_time,
server_name=server_sm.name)
self._hold_enter(db_session)
return (RCODE_OK,
"Master named reconfig done and reconfig queued for all SGs")
def _create_check(self, event):
"""
Process a state machine exception. For putting as much code as
possible on a common call path.
"""
db_session = event.db_session
# Check every half to full holdout time
master_hold_timeout = float(settings['master_hold_timeout'])
delay_factor = (1 + random()) * 0.5
delay_period = timedelta(minutes=delay_factor * master_hold_timeout)
hold_period = timedelta(minutes=master_hold_timeout)
# See if a check event already exists for this ServerSM
current_checks = find_events(ServerSMCheckServer,
db_session,
server_id=self.id_)
current_checks = [e for e in current_checks if e.id_ != event.id_]
if len(current_checks):
return
create_event(ServerSMCheckServer,
db_session=db_session,
sm_id=self.id_,
server_id=self.id_,
delay=delay_period,
server_name=self.name)
def reconfig_master(db_session):
"""
Reconfigure Master DNS server
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMMasterReconfig, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_)
def _reset(self, event):
"""
Reset mastersm
"""
self._init()
# Queue all reconfig
create_event(MasterSMAllReconfig, db_session=event.db_session, sm_id=self.id_, master_id=self.id_)
return (RCODE_OK, "MasterSM - SM reinitialised and MasterSMAllReconfig")
def reconfig_sg(db_session, sg_id, sg_name):
"""
Reconfigure An SGs DNS servers - helper
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMPartialReconfig, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_,
sg_id=sg_id, sg_name=sg_name)
def reconfig_master(db_session):
"""
Reconfigure Master DNS server
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMMasterReconfig,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_)
def reconfig_all(db_session):
"""
Reconfigure all DNS servers - helper
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMAllReconfig,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_)
def reset_master_sm(db_session):
"""
Reset the Configuration state machine
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMReset,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_)
def reconfig_sg(db_session, sg_id, sg_name):
"""
Reconfigure An SGs DNS servers - helper
"""
master_sm = get_master_sm(db_session)
create_event(MasterSMPartialReconfig,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_,
sg_id=sg_id,
sg_name=sg_name)
def _config_change(self, event):
"""
Start the reconfiguration process
"""
self.state = SSTATE_CONFIG
self.retry_msg = None
create_event(ServerSMConfigure,
db_session=event.db_session,
sm_id=self.id_,
server_id=self.id_,
server_name=self.name)
return (RCODE_OK, "Server '%s': reconfiguring" % self.name)
def _reset(self, event):
"""
Reset mastersm
"""
self._init()
# Queue all reconfig
create_event(MasterSMAllReconfig,
db_session=event.db_session,
sm_id=self.id_,
master_id=self.id_)
return (RCODE_OK,
"MasterSM - SM reinitialised and MasterSMAllReconfig")
def _hold_enter(self, db_session, all_reconfig=False):
"""
Enter hold state
Set fields as needed.
"""
if self.state == MSTATE_HOLD:
return
self.hold_sg = HOLD_SG_ALL if all_reconfig else HOLD_SG_NONE
self.state = MSTATE_HOLD
self.hold_start = db_time(db_session)
delay = timedelta(minutes=float(settings["master_hold_timeout"]))
self.hold_stop = self.hold_start + delay
# Queue hold timeout
create_event(MasterSMHoldTimeout, db_session=db_session, sm_id=self.id_, master_id=self.id_, delay=delay)
def zone_sm_dnssec_schedule(db_session, zone_sm, operation):
"""
Schedule a DNSSEC rndc sign/loadkeys operation for a zone_sm
"""
master_sm = get_master_sm(db_session)
if operation == 'loadkeys':
create_event(MasterSMLoadKeys, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_,
zone_name=zone_sm.name)
elif operation in ('sign', 'signzone'):
create_event(MasterSMSignZone, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_,
zone_name=zone_sm.name)
else:
log_error("MasterSM - zone '%s', invalid dnssec operation"
% zone_sm.name)
def zone_sm_reconfig_schedule(db_session, zone_sm, zone_sm_event=None,
randomize=False, master_reconfig=False, **kwargs):
"""
Schedule MasterSM zone creation/update events
Zone SM helper function
"""
master_sm = get_master_sm(db_session)
# According to sampling theorem, twice tick rate, add 1 to account for
# safety
coalesce_time = timedelta(seconds=3*float(settings['sleep_time']))
# Make delay_secs 2 * coalesce time
delay_secs = 6*float(settings['sleep_time'])
if randomize:
delay_secs += 60*float(settings['master_hold_timeout']) * random()
delay_time = timedelta(seconds=delay_secs)
sg_id = zone_sm.sg.id_
# Queue config event
if master_reconfig:
if (master_sm.state != MSTATE_HOLD
or (master_sm.state == MSTATE_HOLD
and master_sm.hold_sg == HOLD_SG_NONE)):
create_event(MasterSMMasterReconfig, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_)
# Only create MasterSMPartialEvents when needed, as they clog the
# event queue
elif (master_sm.state != MSTATE_HOLD
or (master_sm.state == MSTATE_HOLD
and (master_sm.hold_sg != sg_id
and master_sm.hold_sg != HOLD_SG_ALL))):
create_event(MasterSMPartialReconfig, db_session=db_session,
sm_id=master_sm.id_, master_id=master_sm.id_,
sg_id=sg_id, sg_name=zone_sm.sg.name)
# Queue zone_sm event
if not zone_sm_event:
return
if master_sm.state == MSTATE_READY:
create_event(zone_sm_event, db_session=db_session,
sm_id=zone_sm.id_, zone_id=zone_sm.id_,
name=zone_sm.name,
delay=delay_time, coalesce_period=coalesce_time,
**kwargs)
return
elif master_sm.state == MSTATE_HOLD:
schedule_time = master_sm.hold_stop + delay_time
create_event(zone_sm_event, db_session=db_session,
time=schedule_time, coalesce_period=coalesce_time,
sm_id=zone_sm.id_, zone_id=zone_sm.id_,
name=zone_sm.name, **kwargs)
return
else:
log_critical('MasterSM - unrecognized state, exiting')
systemd_exit(os.EX_SOFTWARE, SDEX_GENERIC)
return
def _hold_enter(self, db_session, all_reconfig=False):
"""
Enter hold state
Set fields as needed.
"""
if self.state == MSTATE_HOLD:
return
self.hold_sg = HOLD_SG_ALL if all_reconfig else HOLD_SG_NONE
self.state = MSTATE_HOLD
self.hold_start = db_time(db_session)
delay = timedelta(minutes=float(settings['master_hold_timeout']))
self.hold_stop = self.hold_start + delay
# Queue hold timeout
create_event(MasterSMHoldTimeout,
db_session=db_session,
sm_id=self.id_,
master_id=self.id_,
delay=delay)
def zone_sm_dnssec_schedule(db_session, zone_sm, operation):
"""
Schedule a DNSSEC rndc sign/loadkeys operation for a zone_sm
"""
master_sm = get_master_sm(db_session)
if operation == 'loadkeys':
create_event(MasterSMLoadKeys,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_,
zone_name=zone_sm.name)
elif operation in ('sign', 'signzone'):
create_event(MasterSMSignZone,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_,
zone_name=zone_sm.name)
else:
log_error("MasterSM - zone '%s', invalid dnssec operation" %
zone_sm.name)
def _enable(self, event):
"""
Enable the Server
"""
try:
query = event.db_session.query(ServerSM)\
.filter(ServerSM.address == self.address)\
.filter(ServerSM.state != SSTATE_DISABLED)
result = query.all()
if result:
raise ServerEnableFailure(
"Server '%s' - server '%s' with same address enabled" %
(self.name, result[0].name))
except NoResultFound:
pass
self.state = SSTATE_CONFIG
create_event(ServerSMConfigure,
db_session=event.db_session,
sm_id=self.id_,
server_id=self.id_,
server_name=self.name)
return (RCODE_OK, "Server '%s': enabling" % self.name)
def _hold_time_out(self, event):
"""
Process a hold time out.
This the event runs the associated SM backend routine depending on
value of self.hold_sg
"""
db_session = event.db_session
sm_id = event.py_parameters["sm_id"]
old_hold_sg = self.hold_sg
old_hold_sg_name = self.hold_sg_name
# Reset All SM fields
self._init()
if old_hold_sg == HOLD_SG_ALL:
create_event(MasterSMAllReconfig, sm_id=self.id_, master_id=self.id_)
return (
RCODE_OK,
"MasterSM - %s, %s created and queued" % (MasterSMHoldTimeout.__name__, MasterSMAllReconfig.__name__),
)
if old_hold_sg == HOLD_SG_MASTER:
create_event(MasterSMMasterReconfig, sm_id=self.id_, master_id=self.id_)
return (
RCODE_OK,
"MasterSM - %s, %s created and queued"
% (MasterSMHoldTimeout.__name__, MasterSMMasterReconfig.__name__),
)
elif old_hold_sg:
create_event(
MasterSMPartialReconfig, sm_id=self.id_, master_id=self.id_, sg_id=old_hold_sg, sg_name=old_hold_sg_name
)
return (
RCODE_OK,
"MasterSM - %s, %s for SG %s(%s) created and queued"
% (
MasterSMHoldTimeout.__name__,
MasterSMPartialReconfig.__name__,
old_hold_sg_name,
self._display_hold_sg(old_hold_sg),
),
)
return (RCODE_NOCHANGE, "MasterSM - no reconfigure event during hold")
def _hold_time_out(self, event):
"""
Process a hold time out.
This the event runs the associated SM backend routine depending on
value of self.hold_sg
"""
db_session = event.db_session
sm_id = event.py_parameters['sm_id']
old_hold_sg = self.hold_sg
old_hold_sg_name = self.hold_sg_name
# Reset All SM fields
self._init()
if old_hold_sg == HOLD_SG_ALL:
create_event(MasterSMAllReconfig,
sm_id=self.id_,
master_id=self.id_)
return (
RCODE_OK, "MasterSM - %s, %s created and queued" %
(MasterSMHoldTimeout.__name__, MasterSMAllReconfig.__name__))
if old_hold_sg == HOLD_SG_MASTER:
create_event(MasterSMMasterReconfig,
sm_id=self.id_,
master_id=self.id_)
return (RCODE_OK, "MasterSM - %s, %s created and queued" %
(MasterSMHoldTimeout.__name__,
MasterSMMasterReconfig.__name__))
elif old_hold_sg:
create_event(MasterSMPartialReconfig,
sm_id=self.id_,
master_id=self.id_,
sg_id=old_hold_sg,
sg_name=old_hold_sg_name)
return (RCODE_OK,
"MasterSM - %s, %s for SG %s(%s) created and queued" %
(MasterSMHoldTimeout.__name__,
MasterSMPartialReconfig.__name__, old_hold_sg_name,
self._display_hold_sg(old_hold_sg)))
return (RCODE_NOCHANGE, "MasterSM - no reconfigure event during hold")
def _ready_partial_reconfig(self, event):
"""
Process a partial reconfig event
"""
# Update server address info
recalc_machine_dns_server_info(event.db_session)
# Update master server configuration
rcode, msg = self._master_rndc(event, 'reconfig')
if rcode != RCODE_OK:
return (rcode, msg)
# Issue reconfig event to SG
db_session = event.db_session
ServerGroup = sql_types['ServerGroup']
# sg_id found in zone_sm and sent here as event parameter
sg_id = event.py_parameters['sg_id']
sg_name = event.py_parameters['sg_name']
try:
sg = db_session.query(ServerGroup)\
.filter(ServerGroup.id_ == sg_id).one()
except NoResultFound as exc:
msg = ("MasterSM: can't find SG %s by id '%s'" % (sg_name, sg_id))
raise CantFindSg(msg)
self.hold_sg_name = sg.name
delay_time = timedelta(
seconds=float(settings['master_rndc_settle_delay']))
# Replica SG reconfigure
replica_sg = self.replica_sg
if (replica_sg and replica_sg is not sg):
try:
replica_sg.write_config(db_session, ServerConfigFileError)
except ServerConfigFileError as exc:
log_error(str(exc))
else:
for server_sm in replica_sg.servers:
if server_sm.is_disabled():
continue
create_event(sql_events['ServerSMConfigChange'],
db_session=event.db_session,
sm_id=server_sm.id_,
server_id=server_sm.id_,
delay=delay_time,
server_name=server_sm.name)
# SG reconfigure
try:
sg.write_config(db_session, ServerConfigFileError)
except ServerConfigFileError as exc:
log_error(str(exc))
else:
# Reconfigure servers in this group
for server_sm in sg.servers:
if server_sm.is_disabled():
continue
create_event(sql_events['ServerSMConfigChange'],
db_session=event.db_session,
sm_id=server_sm.id_,
server_id=server_sm.id_,
delay=delay_time,
server_name=server_sm.name)
self._hold_enter(db_session)
return (
RCODE_OK,
"MasterSM: SG '%s' - master named reconfig done and SG reconfig queued"
% self.hold_sg_name)
def zone_sm_reconfig_schedule(db_session,
zone_sm,
zone_sm_event=None,
randomize=False,
master_reconfig=False,
**kwargs):
"""
Schedule MasterSM zone creation/update events
Zone SM helper function
"""
master_sm = get_master_sm(db_session)
# According to sampling theorem, twice tick rate, add 1 to account for
# safety
coalesce_time = timedelta(seconds=3 * float(settings['sleep_time']))
# Make delay_secs 2 * coalesce time
delay_secs = 6 * float(settings['sleep_time'])
if randomize:
delay_secs += 60 * float(settings['master_hold_timeout']) * random()
delay_time = timedelta(seconds=delay_secs)
sg_id = zone_sm.sg.id_
# Queue config event
if master_reconfig:
if (master_sm.state != MSTATE_HOLD
or (master_sm.state == MSTATE_HOLD
and master_sm.hold_sg == HOLD_SG_NONE)):
create_event(MasterSMMasterReconfig,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_)
# Only create MasterSMPartialEvents when needed, as they clog the
# event queue
elif (master_sm.state != MSTATE_HOLD or
(master_sm.state == MSTATE_HOLD and
(master_sm.hold_sg != sg_id and master_sm.hold_sg != HOLD_SG_ALL))):
create_event(MasterSMPartialReconfig,
db_session=db_session,
sm_id=master_sm.id_,
master_id=master_sm.id_,
sg_id=sg_id,
sg_name=zone_sm.sg.name)
# Queue zone_sm event
if not zone_sm_event:
return
if master_sm.state == MSTATE_READY:
create_event(zone_sm_event,
db_session=db_session,
sm_id=zone_sm.id_,
zone_id=zone_sm.id_,
name=zone_sm.name,
delay=delay_time,
coalesce_period=coalesce_time,
**kwargs)
return
elif master_sm.state == MSTATE_HOLD:
schedule_time = master_sm.hold_stop + delay_time
create_event(zone_sm_event,
db_session=db_session,
time=schedule_time,
coalesce_period=coalesce_time,
sm_id=zone_sm.id_,
zone_id=zone_sm.id_,
name=zone_sm.name,
**kwargs)
return
else:
log_critical('MasterSM - unrecognized state, exiting')
systemd_exit(os.EX_SOFTWARE, SDEX_GENERIC)
return
def _ready_partial_reconfig(self, event):
"""
Process a partial reconfig event
"""
# Update server address info
recalc_machine_dns_server_info(event.db_session)
# Update master server configuration
rcode, msg = self._master_rndc(event, 'reconfig')
if rcode != RCODE_OK:
return (rcode, msg)
# Issue reconfig event to SG
db_session = event.db_session
ServerGroup = sql_types['ServerGroup']
# sg_id found in zone_sm and sent here as event parameter
sg_id = event.py_parameters['sg_id']
sg_name = event.py_parameters['sg_name']
try:
sg = db_session.query(ServerGroup)\
.filter(ServerGroup.id_ == sg_id).one()
except NoResultFound as exc:
msg = ("MasterSM: can't find SG %s by id '%s'"
% (sg_name, sg_id))
raise CantFindSg(msg)
self.hold_sg_name = sg.name
delay_time = timedelta(
seconds=float(settings['master_rndc_settle_delay']))
# Replica SG reconfigure
replica_sg = self.replica_sg
if (replica_sg and replica_sg is not sg):
try:
replica_sg.write_config(db_session, ServerConfigFileError)
except ServerConfigFileError as exc:
log_error(str(exc))
else:
for server_sm in replica_sg.servers:
if server_sm.is_disabled():
continue
create_event(sql_events['ServerSMConfigChange'],
db_session=event.db_session,
sm_id=server_sm.id_, server_id=server_sm.id_,
delay=delay_time,
server_name=server_sm.name)
# SG reconfigure
try:
sg.write_config(db_session, ServerConfigFileError)
except ServerConfigFileError as exc:
log_error(str(exc))
else:
# Reconfigure servers in this group
for server_sm in sg.servers:
if server_sm.is_disabled():
continue
create_event(sql_events['ServerSMConfigChange'],
db_session=event.db_session,
sm_id=server_sm.id_, server_id=server_sm.id_,
delay=delay_time,
server_name=server_sm.name)
self._hold_enter(db_session)
return (RCODE_OK, "MasterSM: SG '%s' - master named reconfig done and SG reconfig queued"
% self.hold_sg_name)
