def vacuum_syslog (self, age_days=None):
"""
Destroy syslog messages received more than age_days ago
"""
self._begin_op()
db_session = self.db_session
if age_days is None:
age_days = float(zone_cfg.get_row_exc(db_session,
key='syslog_max_age'))
age_days = timedelta(days=age_days)
count = 0
# Do a straight SQL DELETE first to speed things along
# Count events to be deleted
syslog_table = sql_data['tables'][SyslogMsg]
where_stmt = and_(SyslogMsg.receivedat != None,
(func.now() - SyslogMsg.receivedat) > age_days)
count_select = select([func.count(syslog_table.c.get('id'))],
where_stmt)
result = db_session.execute(count_select).fetchall()
count += result[0][0]
db_session.execute(syslog_table.delete().where(where_stmt))
result = {'num_deleted': count}
self._finish_op()
return result
def vacuum_event_queue(self, age_days=None):
"""
Destroy events processed more than age_days ago
"""
self._begin_op()
db_session = self.db_session
if age_days is None:
age_days = float(zone_cfg.get_row_exc(db_session,
key='event_max_age'))
age_days = timedelta(days=age_days)
count = 0
# Do a straight SQL DELETE first to speed things along
# Count events to be deleted
event_table = sql_data['tables'][Event]
where_stmt = and_(Event.state.in_(event_processed_states),
Event.processed != None,
(func.now() - Event.processed) > age_days)
count_select = select([func.count(event_table.c.get('id'))],
where_stmt)
result = db_session.execute(count_select).fetchall()
count += result[0][0]
db_session.execute(event_table.delete().where(where_stmt))
result = {'num_deleted': count}
self._finish_op()
return result
def set_missing_zi_data():
"""
Set missing fields in supplied zi_data to prevent problems
"""
# Set ZI Zone ttl if not already set
if 'zone_ttl' not in zi_data:
zi_data['zone_ttl'] = zone_ttl
# Set other SOA values in zi_data from defaults
# if they are not there. soa_ttl can be None
for field in [
'soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry',
'soa_expire', 'soa_minimum'
]:
if not zi_data.get(field):
zi_data[field] = zone_cfg.get_row_exc(db_session,
field,
sg=zone_sm.sg)
# We always update serial number on zone udpdate/publish
# but it is nicer and probably less troublesome to replace
# an existing serial number that may be out there
if not zi_data.get('soa_serial'):
if zone_sm.soa_serial:
zi_data['soa_serial'] = zone_sm.soa_serial
else:
# Obviously a new zone
zi_data['soa_serial'] = new_zone_soa_serial(db_session)
def get_default_zi_data(db_session, sg_name=None):
"""
Return default zi data from zone_cfg table
This is called from wsgi code or zone_tool
"""
zi_data = {}
soa_fields = ['soa_mname', 'soa_rname', 'soa_refresh',
'soa_retry', 'soa_expire', 'soa_minimum']
for field in soa_fields:
zi_data[field] = zone_cfg.get_row_exc(db_session, field,
sg_name=sg_name)
zi_data['zone_ttl'] = zone_cfg.get_row_exc(db_session, 'zone_ttl',
sg_name=sg_name)
zi_data['soa_ttl'] = None
zi_data['soa_serial'] = new_zone_soa_serial(db_session)
return zi_data
def vacuum_zis(self, age_days=None, zi_max_num=None):
"""
Age ZIs according to age_days and zi_max_num
"""
self._begin_op()
db_session = self.db_session
db_query_slice = get_numeric_setting('db_query_slice', int)
if age_days is None:
age_days = float(zone_cfg.get_row_exc(db_session,
key='zi_max_age'))
age_days = timedelta(days=age_days)
if zi_max_num is None:
zi_max_num = int(zone_cfg.get_row_exc(db_session,
key='zi_max_num'))
stmt = db_session.query(ZoneInstance.zone_id,
func.count(ZoneInstance.id_).label('zi_count'))\
.group_by(ZoneInstance.zone_id).subquery()
zone_sm_query = db_session.query(ZoneSM)\
.filter(ZoneSM.state != ZSTATE_DELETED)\
.outerjoin(stmt, ZoneSM.id_ == stmt.c.zone_id)\
.filter(stmt.c.zi_count > zi_max_num)\
.yield_per(db_query_slice)
count = 0
for zone_sm in zone_sm_query:
zi_keep = db_session.query(ZoneInstance.id_)\
.filter(ZoneInstance.zone_id == zone_sm.id_)\
.order_by(desc(ZoneInstance.mtime))\
.limit(zi_max_num)
zi_query = db_session.query(ZoneInstance)\
.filter(ZoneInstance.zone_id == zone_sm.id_)\
.filter(ZoneInstance.id_ != zone_sm.zi_id)\
.filter(not_(ZoneInstance.id_.in_(zi_keep)))\
.filter(ZoneInstance.mtime < (func.now() - age_days))
for zi in zi_query:
if (zi.id_ == zone_sm.zi_id
or zi.id_ == zone_sm.zi_candidate_id):
# Skip if this ZI has ben selected for republishing in
# the mean time
continue
db_session.delete(zi)
count += 1
result = {'num_deleted': count}
self._finish_op()
return result
def vacuum_zones(self, age_days=None):
"""
Destroy zones older than age_days
"""
self._begin_op()
db_session = self.db_session
db_query_slice = get_numeric_setting('db_query_slice', int)
age_days_from_config = float(zone_cfg.get_row_exc(db_session,
key='zone_del_age'))
if age_days_from_config <= 0 and age_days is None:
age_days = get_numeric_setting('zone_del_off_age', float)
elif age_days is None:
age_days = age_days_from_config
age_days = timedelta(days=age_days)
count = 0
# Clear old and nuked zones one by one
id_query = db_session.query(ZoneSM.id_)\
.filter(ZoneSM.state == ZSTATE_DELETED)\
.filter(or_(ZoneSM.deleted_start == None,
(func.now() - ZoneSM.deleted_start) > age_days))\
.filter(ZoneSM.zone_files == False)\
.yield_per(db_query_slice)
id_results = []
for zone_id, in id_query:
id_results.append(zone_id)
for zone_id in id_results:
try:
zone_sm = db_session.query(ZoneSM)\
.filter(ZoneSM.id_ == zone_id).one()
except NoResultFound:
continue
if zone_sm.state != ZSTATE_DELETED:
# Skip this if a customer has undeleted zone in the mean time..
continue
db_session.delete(zone_sm)
db_session.commit()
count += 1
# Finally do zone_sm destroy operation to
query = db_session.query(ZoneSM)\
.filter(ZoneSM.state == ZSTATE_DELETED)\
.filter(or_(ZoneSM.deleted_start == None,
(func.now() - ZoneSM.deleted_start) > age_days))
for zone_sm in query:
if zone_sm.state != ZSTATE_DELETED:
# Skip this if a customer has undeleted zone in the mean time..
continue
try:
exec_zonesm(zone_sm, ZoneSMDoDestroy)
except ZoneSmFailure:
continue
count += 1
result = {'num_deleted': count}
self._finish_op()
return result
def get_default_zi_data(db_session, sg_name=None):
"""
Return default zi data from zone_cfg table
This is called from wsgi code or zone_tool
"""
zi_data = {}
soa_fields = [
'soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire',
'soa_minimum'
]
for field in soa_fields:
zi_data[field] = zone_cfg.get_row_exc(db_session,
field,
sg_name=sg_name)
zi_data['zone_ttl'] = zone_cfg.get_row_exc(db_session,
'zone_ttl',
sg_name=sg_name)
zi_data['soa_ttl'] = None
zi_data['soa_serial'] = new_zone_soa_serial(db_session)
return zi_data
def move_server_sg(db_session, server_name, sg_name=None):
"""
Move a server between SGs
"""
server_sm = find_server_byname(db_session, server_name)
if not server_sm.is_disabled():
raise ServerNotDisabled(server_sm.name)
if not sg_name:
sg_name = zone_cfg.get_row_exc(db_session, 'default_sg')
if not sg_name in list_all_sgs(db_session):
raise NoSgFound(sg_name)
sg = find_sg_byname(db_session, sg_name, raise_exc=True)
server_sm.set_sg(sg)
def vacuum_pare_deleted_zone_zis(self, age_days=None):
"""
Pare ZIs on deleted zones older than age_days
"""
self._begin_op()
db_session = self.db_session
db_query_slice = get_numeric_setting('db_query_slice', int)
age_days_from_config = float(zone_cfg.get_row_exc(db_session,
key='zone_del_pare_age'))
if age_days_from_config <= 0 and age_days is None:
return {'num_deleted': 0}
if age_days is None:
age_days = age_days_from_config
age_days = timedelta(days=age_days)
stmt = db_session.query(ZoneInstance.zone_id,
func.count(ZoneInstance.id_).label('zi_count'))\
.group_by(ZoneInstance.zone_id).subquery()
zone_sm_query = db_session.query(ZoneSM)\
.filter(ZoneSM.state == ZSTATE_DELETED)\
.outerjoin(stmt, ZoneSM.id_ == stmt.c.zone_id)\
.filter(stmt.c.zi_count > 1)\
.filter(and_(ZoneSM.deleted_start != None,
(func.now() - ZoneSM.deleted_start) > age_days))\
.yield_per(db_query_slice)
count = 0
for zone_sm in zone_sm_query:
zi_query = db_session.query(ZoneInstance)\
.filter(ZoneInstance.zone_id == zone_sm.id_)\
.filter(ZoneInstance.id_ != zone_sm.zi_id)
if zone_sm.state != ZSTATE_DELETED:
# Skip this if a customer has undeleted zone in the mean time..
continue
for zi in zi_query:
if (zi.id_ == zone_sm.zi_id
or zi.id_ == zone_sm.zi_candidate_id):
# Skip if this ZI has published or selected to be published
continue
db_session.delete(zi)
count += 1
result = {'num_deleted': count}
self._finish_op()
return result
def new_server(db_session,
server_name,
address,
sg_name,
server_type=None,
ssh_address=None):
"""
Create a new server
"""
server_name = server_name.lower()
if server_name.endswith('.'):
server_name = server_name[:-1]
if not sg_name:
sg_name = zone_cfg.get_row_exc(db_session, 'default_sg')
if not sg_name in list_all_sgs(db_session):
raise NoSgFound(sg_name)
try:
server_list = db_session.query(ServerSM)\
.filter(ServerSM.name == server_name).all()
if len(server_list):
raise ServerExists(server_name)
except NoResultFound:
pass
if not server_type:
server_type = zone_cfg.get_row(db_session,
'default_stype',
raise_exc=True)
server_sm = ServerSM(server_name, address, sg_name, server_type,
ssh_address)
try:
db_session.add(server_sm)
db_session.flush()
except IntegrityError as exc:
raise ServerAddressExists(address)
sg = find_sg_byname(db_session, sg_name, raise_exc=True)
server_sm.set_sg(sg)
db_session.flush()
return server_sm
def set_missing_zi_data():
"""
Set missing fields in supplied zi_data to prevent problems
"""
# Set ZI Zone ttl if not already set
if 'zone_ttl' not in zi_data:
zi_data['zone_ttl'] = zone_ttl
# Set other SOA values in zi_data from defaults
# if they are not there. soa_ttl can be None
for field in ['soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry',
'soa_expire', 'soa_minimum']:
if not zi_data.get(field):
zi_data[field] = zone_cfg.get_row_exc(db_session, field,
sg=zone_sm.sg)
# We always update serial number on zone udpdate/publish
# but it is nicer and probably less troublesome to replace
# an existing serial number that may be out there
if not zi_data.get('soa_serial'):
if zone_sm.soa_serial:
zi_data['soa_serial'] = zone_sm.soa_serial
else:
# Obviously a new zone
zi_data['soa_serial'] = new_zone_soa_serial(db_session)
def _data_to_zi(self, name, zi_data, change_by, normalize_ttls=False,
admin_privilege=False, helpdesk_privilege=False):
"""
Construct a new ZI, RRS and comments, from zone_data.
"""
def set_missing_zi_data():
"""
Set missing fields in supplied zi_data to prevent problems
"""
# Set ZI Zone ttl if not already set
if 'zone_ttl' not in zi_data:
zi_data['zone_ttl'] = zone_ttl
# Set other SOA values in zi_data from defaults
# if they are not there. soa_ttl can be None
for field in ['soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry',
'soa_expire', 'soa_minimum']:
if not zi_data.get(field):
zi_data[field] = zone_cfg.get_row_exc(db_session, field,
sg=zone_sm.sg)
# We always update serial number on zone udpdate/publish
# but it is nicer and probably less troublesome to replace
# an existing serial number that may be out there
if not zi_data.get('soa_serial'):
if zone_sm.soa_serial:
zi_data['soa_serial'] = zone_sm.soa_serial
else:
# Obviously a new zone
zi_data['soa_serial'] = new_zone_soa_serial(db_session)
def check_zi_data():
"""
Check incoming zi_data attributes for correctness
"""
for field in ['soa_mname', 'soa_rname']:
validate_zi_hostname(name, field, zi_data[field])
for field in ['soa_refresh', 'soa_retry', 'soa_expire',
'soa_minimum', 'soa_ttl', 'zone_ttl']:
if field == 'soa_ttl' and not zi_data.get(field):
# SOA TTL can be None
continue
validate_zi_ttl(name, field, zi_data[field])
for field in ['soa_serial']:
if field == 'soa_serial' and zi_data.get(field, None) == None:
# SOA serial can be None
continue
# Check incoming data type of soa_serial
if not isinstance(zi_data['soa_serial'], int):
raise SOASerialTypeError(name)
if not ( 0 < zi_data['soa_serial'] <= (2**32-1)):
# RFC 2136 Section 4.2 AO serial cannot be zero
raise SOASerialRangeError(name)
# Function start
db_session = self.db_session
# Get zone_sm to get zone ID etc
zone_sm = self._get_zone_sm(name)
zone_id = zone_sm.id_
# initialise data and zone consistency checking
data_tools = DataTools(db_session, zone_sm)
# Sort out a candidate value for zone_ttl so that RRs can be created
zone_ttl = zi_data.get('zone_ttl',
zone_cfg.get_row_exc(db_session, 'zone_ttl', sg=zone_sm.sg))
zone_ttl_supplied = 'zone_ttl' in zi_data
# Create comments, and set up comment IDs, and stuff for handlng
# RR Groups zi_data structures
data_tools.rr_data_create_comments(zi_data, zone_ttl)
# Deal with ZI data problems, and supply defaults if missing
set_missing_zi_data()
check_zi_data()
# This constructor call sets attributes in zi as well!
zi = ZoneInstance(change_by=change_by, **zi_data)
db_session.add(zi)
apex_comment = data_tools.get_apex_comment()
if apex_comment:
zi.add_apex_comment(apex_comment)
# Get zi.id_ zi.zone_id from database
db_session.flush()
# Add RRs to zi
# Note use of lambda so that list of rrs is always refreshed in
# function
data_tools.add_rrs(lambda :zi.rrs, zi.add_rr,
admin_privilege, helpdesk_privilege)
# tie zi into data_structures
zone_sm.all_zis.append(zi)
zi.zone = zone_sm
db_session.flush()
# Normalise TTLs here
if normalize_ttls and zone_ttl_supplied:
zi.normalize_ttls()
# Update SOA and NS records - can't hurt to do it here
# This also cleans out any incoming apex NS records if
# client should not be setting them.
zi.update_apex(db_session)
# Update Zone TTLs for clean initialisation
zi.update_zone_ttls()
db_session.flush()
# Check zone consistency. Do this here as Apex RRs need to be complete.
data_tools.check_zi_consistency(zi.rrs)
return zi, data_tools.get_auto_ptr_data()
def get_config_default(self, config_key):
"""
Get the default value for a configuration key
"""
self.refresh_db_session()
return zone_cfg.get_row_exc(self.db_session, config_key)
def _data_to_zi(self,
name,
zi_data,
change_by,
normalize_ttls=False,
admin_privilege=False,
helpdesk_privilege=False):
"""
Construct a new ZI, RRS and comments, from zone_data.
"""
def set_missing_zi_data():
"""
Set missing fields in supplied zi_data to prevent problems
"""
# Set ZI Zone ttl if not already set
if 'zone_ttl' not in zi_data:
zi_data['zone_ttl'] = zone_ttl
# Set other SOA values in zi_data from defaults
# if they are not there. soa_ttl can be None
for field in [
'soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry',
'soa_expire', 'soa_minimum'
]:
if not zi_data.get(field):
zi_data[field] = zone_cfg.get_row_exc(db_session,
field,
sg=zone_sm.sg)
# We always update serial number on zone udpdate/publish
# but it is nicer and probably less troublesome to replace
# an existing serial number that may be out there
if not zi_data.get('soa_serial'):
if zone_sm.soa_serial:
zi_data['soa_serial'] = zone_sm.soa_serial
else:
# Obviously a new zone
zi_data['soa_serial'] = new_zone_soa_serial(db_session)
def check_zi_data():
"""
Check incoming zi_data attributes for correctness
"""
for field in ['soa_mname', 'soa_rname']:
validate_zi_hostname(name, field, zi_data[field])
for field in [
'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum',
'soa_ttl', 'zone_ttl'
]:
if field == 'soa_ttl' and not zi_data.get(field):
# SOA TTL can be None
continue
validate_zi_ttl(name, field, zi_data[field])
for field in ['soa_serial']:
if field == 'soa_serial' and zi_data.get(field, None) == None:
# SOA serial can be None
continue
# Check incoming data type of soa_serial
if not isinstance(zi_data['soa_serial'], int):
raise SOASerialTypeError(name)
if not (0 < zi_data['soa_serial'] <= (2**32 - 1)):
# RFC 2136 Section 4.2 AO serial cannot be zero
raise SOASerialRangeError(name)
# Function start
db_session = self.db_session
# Get zone_sm to get zone ID etc
zone_sm = self._get_zone_sm(name)
zone_id = zone_sm.id_
# initialise data and zone consistency checking
data_tools = DataTools(db_session, zone_sm)
# Sort out a candidate value for zone_ttl so that RRs can be created
zone_ttl = zi_data.get(
'zone_ttl',
zone_cfg.get_row_exc(db_session, 'zone_ttl', sg=zone_sm.sg))
zone_ttl_supplied = 'zone_ttl' in zi_data
# Create comments, and set up comment IDs, and stuff for handlng
# RR Groups zi_data structures
data_tools.rr_data_create_comments(zi_data, zone_ttl)
# Deal with ZI data problems, and supply defaults if missing
set_missing_zi_data()
check_zi_data()
# This constructor call sets attributes in zi as well!
zi = ZoneInstance(change_by=change_by, **zi_data)
db_session.add(zi)
apex_comment = data_tools.get_apex_comment()
if apex_comment:
zi.add_apex_comment(apex_comment)
# Get zi.id_ zi.zone_id from database
db_session.flush()
# Add RRs to zi
# Note use of lambda so that list of rrs is always refreshed in
# function
data_tools.add_rrs(lambda: zi.rrs, zi.add_rr, admin_privilege,
helpdesk_privilege)
# tie zi into data_structures
zone_sm.all_zis.append(zi)
zi.zone = zone_sm
db_session.flush()
# Normalise TTLs here
if normalize_ttls and zone_ttl_supplied:
zi.normalize_ttls()
# Update SOA and NS records - can't hurt to do it here
# This also cleans out any incoming apex NS records if
# client should not be setting them.
zi.update_apex(db_session)
# Update Zone TTLs for clean initialisation
zi.update_zone_ttls()
db_session.flush()
# Check zone consistency. Do this here as Apex RRs need to be complete.
data_tools.check_zi_consistency(zi.rrs)
return zi, data_tools.get_auto_ptr_data()
