def vacuum_zones(self, age_days=None):
"""
Destroy zones older than age_days
"""
self._begin_op()
db_session = self.db_session
db_query_slice = get_numeric_setting('db_query_slice', int)
age_days_from_config = float(zone_cfg.get_row_exc(db_session,
key='zone_del_age'))
if age_days_from_config <= 0 and age_days is None:
age_days = get_numeric_setting('zone_del_off_age', float)
elif age_days is None:
age_days = age_days_from_config
age_days = timedelta(days=age_days)
count = 0
# Clear old and nuked zones one by one
id_query = db_session.query(ZoneSM.id_)\
.filter(ZoneSM.state == ZSTATE_DELETED)\
.filter(or_(ZoneSM.deleted_start == None,
(func.now() - ZoneSM.deleted_start) > age_days))\
.filter(ZoneSM.zone_files == False)\
.yield_per(db_query_slice)
id_results = []
for zone_id, in id_query:
id_results.append(zone_id)
for zone_id in id_results:
try:
zone_sm = db_session.query(ZoneSM)\
.filter(ZoneSM.id_ == zone_id).one()
except NoResultFound:
continue
if zone_sm.state != ZSTATE_DELETED:
# Skip this if a customer has undeleted zone in the mean time..
continue
db_session.delete(zone_sm)
db_session.commit()
count += 1
# Finally do zone_sm destroy operation to
query = db_session.query(ZoneSM)\
.filter(ZoneSM.state == ZSTATE_DELETED)\
.filter(or_(ZoneSM.deleted_start == None,
(func.now() - ZoneSM.deleted_start) > age_days))
for zone_sm in query:
if zone_sm.state != ZSTATE_DELETED:
# Skip this if a customer has undeleted zone in the mean time..
continue
try:
exec_zonesm(zone_sm, ZoneSMDoDestroy)
except ZoneSmFailure:
continue
count += 1
result = {'num_deleted': count}
self._finish_op()
return result
def reset_all_zones(self):
"""
Reset all zones
"""
self._begin_op()
db_session = self.db_session
id_query = db_session.query(ZoneSM.id_, ZoneSM.name)
id_query = ZoneSM.query_is_not_disabled_deleted(id_query)
id_result = id_query.all()
for zone_id, zone_name in id_result:
try:
zone_sm = db_session.query(ZoneSM)\
.filter(ZoneSM.id_ == zone_id).one()
except NoResultFound:
raise ZoneNotFoundByZoneId(zone_id)
exec_zonesm(zone_sm, ZoneSMDoReset)
self._finish_op()
def refresh_sg(self, sg_name):
"""
Refresh all zones on an SG
"""
self._begin_op()
db_session = self.db_session
sg = self._find_sg_byname(sg_name)
id_query = db_session.query(ZoneSM.id_, ZoneSM.name)\
.filter(ZoneSM.sg_id == sg.id_)
id_query = ZoneSM.query_is_configured(id_query)
id_result = id_query.all()
for zone_id, zone_name in id_result:
try:
zone_sm = db_session.query(ZoneSM)\
.filter(ZoneSM.id_ == zone_id).one()
except NoResultFound:
raise ZoneNotFoundByZoneId(zone_id)
exec_zonesm(zone_sm, ZoneSMDoRefresh)
self._finish_op()
def _queue_auto_ptr_data(self, auto_ptr_data):
"""
Queue auto PTR data as incremental updates against respective reverse
zones.
"""
if not auto_ptr_data:
return
if not len(auto_ptr_data):
return
if not settings['auto_reverse']:
return
db_session = self.db_session
# Create new update_group
ug_dict = {}
auto_ptr_privilege_flag = False
for ptr_data in auto_ptr_data:
# Ignore addresses we don't have reverse zone for
query = db_session.query(ZoneSM)\
.join(ReverseNetwork)\
.filter(":address <<= reverse_networks.network")\
.params(address = ptr_data['address'])
query = ZoneSM.query_is_not_deleted(query)
query = ZoneSM.query_inc_updates(query)
query = query.order_by(ReverseNetwork.network.desc())\
.limit(1)
try:
zone_sm = query.one()
except NoResultFound:
continue
# Ignore invalid host names
if not is_inet_hostname(ptr_data['hostname'], absolute=True,
wildcard=False):
log_error("Hostname '%s' is not a valid hostname."
% ptr_data['hostname'])
continue
# Determine proposed update operation
update_op = RROP_PTR_UPDATE_FORCE if ptr_data['force_reverse'] \
else RROP_PTR_UPDATE
# Execute privilege checks ahead of time to save unnecessary churn
# Better than needlessly going through whole rigamorole of
# incremental update processing later on for no effect
#1 See if old PTR exists to retrieve any RR reference
# Both following also used lower down when generating RR_PTR
label = label_from_address(ptr_data['address'])
rr_ref = find_reference(db_session, ptr_data['reference'],
raise_exc=False)
# query for old record - this generates one select
# Optimization - if check has previously suceeded, don't check
# again as this is all checked further in
if not auto_ptr_privilege_flag:
qlabel= label[:label.rfind(zone_sm.name)-1]
query = db_session.query(ResourceRecord)\
.filter(ResourceRecord.label == qlabel)\
.filter(ResourceRecord.zi_id == zone_sm.zi_candidate_id)\
.filter(ResourceRecord.disable == False)\
.filter(ResourceRecord.type_ == RRTYPE_PTR)
old_rrs = query.all()
old_rr = old_rrs[0] if len(old_rrs) else None
# Check that we can proceed, only if check has not succeded yet
if not check_auto_ptr_privilege(rr_ref, self.sectag, zone_sm,
old_rr):
if old_rr:
log_debug("Zone '%s' - can't replace '%s' PTR"
" as neither"
" sectags '%s' vs '%s'"
" references '%s' vs '%s'/'%s' (old PTR/rev zone)"
"match ,"
" or values not given."
% (zone_sm.name, old_rr.label,
self.sectag.sectag, settings['admin_sectag'],
rr_ref, old_rr.reference, zone_sm.reference))
else:
log_debug("Zone '%s' - can't add '%s' PTR as neither"
" sectags '%s' vs '%s'"
" references '%s' vs '%s' (rev zone) match,"
" or values not given."
% (zone_sm.name, qlabel,
self.sectag.sectag, settings['admin_sectag'],
rr_ref, zone_sm.reference))
continue
auto_ptr_privilege_flag = True
# Create a new update group if zone has not been seen before.
try:
update_group, zone_ttl = ug_dict.get(zone_sm)
except (ValueError, TypeError):
# Obtain reverse zone_ttl so PTR rrs can be created
# Use candidate ZI as it always is available.
# zi is published zi
zi = self._get_zi(zone_sm.zi_candidate_id)
if not zi:
log_error("Zone '%s': does not have candidate zi."
% zone_sm.name)
continue
zone_ttl = zi.zone_ttl
update_group = new_update_group(db_session, None,
zone_sm, None, ptr_only=True,
sectag=self.sectag.sectag)
ug_dict[zone_sm] = (update_group, zone_ttl)
# Allocate RR_PTR update record
rr = RR_PTR(label=label, zone_ttl=zone_ttl,
rdata=ptr_data['hostname'], disable=ptr_data['disable'],
domain=zone_sm.name, update_op=update_op)
rr.ref_id = rr_ref.id_ if rr_ref else None
rr.reference = rr_ref
# Chain on RR_PTR update record
update_group.update_ops.append(rr)
# Flush everything to disk
db_session.flush()
# Issue zone refreshes to implement PTR changes
for zone_sm in ug_dict:
if zone_sm.is_disabled():
continue
exec_zonesm(zone_sm, ZoneSMDoRefresh)
# Make sure everything is committed
db_session.commit()
def _data_to_update(self, name, update_data, update_type, change_by,
admin_privilege=False, helpdesk_privilege=False):
"""
Construct an update group for a zone, from supplied RRS and comments.
Functional equivalent of _data_to_zi() above, but for incremental
updates
"""
# Function start
db_session = self.db_session
# Check that update_type is supplied
if not update_type:
raise UpdateTypeRequired(name)
# Get zone_sm to get zone ID etc
zone_sm = self._get_zone_sm(name)
zone_id = zone_sm.id_
# See if incremental updates are enabled for zone before queuing any
if not zone_sm.inc_updates:
raise IncrementalUpdatesDisabled(name)
# Don't queue updates for a disabled zone
if zone_sm.is_disabled():
raise ZoneDisabled(name)
# Privilege check for no apex zones - admin only
if not zone_sm.use_apex_ns and not admin_privilege:
raise ZoneAdminPrivilegeNeeded(name)
# Use candidate ZI as it always is available. zi is published zi
zi = self._get_zi(zone_sm.zi_candidate_id)
if not zi:
raise ZiNotFound(name, zone_sm.zi_candidate_id)
# Get value of zone_ttl so that RRs can be created
zone_ttl = zi.zone_ttl
# Create RRs list from published ZI
pzi = PseudoZi(db_session, zi)
# initialise data and zone consistency checking
zi_cname_flag = False
if len([r for r in pzi.rrs if r.type_ == RRTYPE_CNAME]):
zi_cname_flag = True
data_tools = DataTools(db_session, zone_sm, zi_cname_flag)
# Create comments, and set up comment IDs, and stuff for handlng
# RR Groups zi_data structures
data_tools.rr_data_create_comments(update_data, zone_ttl,
creating_real_zi=False)
try:
# Create new update_group
update_group = new_update_group(db_session, update_type,
zone_sm, change_by)
except IntegrityError as exc:
raise UpdateTypeAlreadyQueued(name, update_type)
# Add RRs to DB and operate on Pseudo ZI
data_tools.add_rrs(lambda :pzi.rrs, pzi.trial_op_rr,
admin_privilege, helpdesk_privilege, update_group=update_group)
data_tools.check_zi_consistency(pzi.rrs)
# Get all data out to DB, and ids etc established.
db_session.flush()
# Refresh zone to implement updates
exec_zonesm(zone_sm, ZoneSMDoRefresh)
# Return auto update info
return data_tools.get_auto_ptr_data()
def nuke_zones(self, *names, include_deleted=False, toggle_deleted=False,
sg_name=None, reference=None):
"""
Destroy multiple zones. Multiple names may be given. Wildcards
can be used for partial matches.
This is mainly a command for testing, or cleaning up after a large
batch zone load goes awry.
Zones being nuked have their deleted_start set to 1/1/1970, midnight.
This means they will be immediately reaped by the next vacuum_zones
command.
"""
# Deal with SA auto-BEGIN - want fresh transaction to see fresh data
self._begin_op()
if not names:
# No arguments
self._finish_op()
raise NoZonesFound('')
db_session = self.db_session
db_query_slice = get_numeric_setting('db_query_slice', int)
# We were given some arguments
zones = []
# We keep domains and labels in database lowercase
names = [x.lower() for x in names]
name_pattern = ' '.join(names)
names = [x.replace('*', '%') for x in names]
names = [x.replace('?', '_') for x in names]
for name in names:
if not name.endswith('.') and not name.endswith('%'):
name += '.'
query = db_session.query(ZoneSM)\
.filter(ZoneSM.name.like(name))
# Don't delete any reverse zones with this command
query = query.filter(not_(ZoneSM.name.like('%.in-addr.arpa.')))\
.filter(not_(ZoneSM.name.like('%.ip6.arpa.')))
if reference:
query = query.join(Reference)\
.filter(Reference.reference.ilike(reference))
if sg_name and self.sectag.sectag == settings['admin_sectag']:
if sg_name not in list_all_sgs(self.db_session):
raise NoSgFound(sg_name)
query = query.join(ServerGroup, ZoneSM.sg_id
== ServerGroup.id_)\
.filter(ServerGroup.name == sg_name)
if include_deleted:
pass
elif toggle_deleted:
query = query.filter(ZoneSM.state == ZSTATE_DELETED)
else:
query = query.filter(ZoneSM.state != ZSTATE_DELETED)
query = query.yield_per(db_query_slice)
# The following gives less RAM piggery even though it is slower
for z in query:
zones.append(z)
# Take note of security tags
if self.sectag.sectag != settings['admin_sectag']:
zones = [x for x in zones if self.sectag in x.sectags]
if not zones:
if len(name_pattern) > 240:
name_pattern = '* - %s names' % len(names)
raise NoZonesFound(name_pattern)
# Mark them all as deleted.
for zone in zones:
exec_zonesm(zone, ZoneSMNukeStart)
self._finish_op()
def _queue_auto_ptr_data(self, auto_ptr_data):
"""
Queue auto PTR data as incremental updates against respective reverse
zones.
"""
if not auto_ptr_data:
return
if not len(auto_ptr_data):
return
if not settings['auto_reverse']:
return
db_session = self.db_session
# Create new update_group
ug_dict = {}
auto_ptr_privilege_flag = False
for ptr_data in auto_ptr_data:
# Ignore addresses we don't have reverse zone for
query = db_session.query(ZoneSM)\
.join(ReverseNetwork)\
.filter(text(":address <<= reverse_networks.network"))\
.params(address = ptr_data['address'])
query = ZoneSM.query_is_not_deleted(query)
query = ZoneSM.query_inc_updates(query)
query = query.order_by(ReverseNetwork.network.desc())\
.limit(1)
try:
zone_sm = query.one()
except NoResultFound:
continue
# Ignore invalid host names
if not is_inet_hostname(
ptr_data['hostname'], absolute=True, wildcard=False):
log_error("Hostname '%s' is not a valid hostname." %
ptr_data['hostname'])
continue
# Determine proposed update operation
update_op = RROP_PTR_UPDATE_FORCE if ptr_data['force_reverse'] \
else RROP_PTR_UPDATE
# Execute privilege checks ahead of time to save unnecessary churn
# Better than needlessly going through whole rigamorole of
# incremental update processing later on for no effect
#1 See if old PTR exists to retrieve any RR reference
# Both following also used lower down when generating RR_PTR
label = label_from_address(ptr_data['address'])
rr_ref = find_reference(db_session,
ptr_data['reference'],
raise_exc=False)
# query for old record - this generates one select
# Optimization - if check has previously suceeded, don't check
# again as this is all checked further in
if not auto_ptr_privilege_flag:
qlabel = label[:label.rfind(zone_sm.name) - 1]
query = db_session.query(ResourceRecord)\
.filter(ResourceRecord.label == qlabel)\
.filter(ResourceRecord.zi_id == zone_sm.zi_candidate_id)\
.filter(ResourceRecord.disable == False)\
.filter(ResourceRecord.type_ == RRTYPE_PTR)
old_rrs = query.all()
old_rr = old_rrs[0] if len(old_rrs) else None
# Check that we can proceed, only if check has not succeded yet
if not check_auto_ptr_privilege(rr_ref, self.sectag, zone_sm,
old_rr):
if old_rr:
log_debug(
"Zone '%s' - can't replace '%s' PTR"
" as neither"
" sectags '%s' vs '%s'"
" references '%s' vs '%s'/'%s' (old PTR/rev zone)"
"match ,"
" or values not given." %
(zone_sm.name, old_rr.label, self.sectag.sectag,
settings['admin_sectag'], rr_ref,
old_rr.reference, zone_sm.reference))
else:
log_debug("Zone '%s' - can't add '%s' PTR as neither"
" sectags '%s' vs '%s'"
" references '%s' vs '%s' (rev zone) match,"
" or values not given." %
(zone_sm.name, qlabel, self.sectag.sectag,
settings['admin_sectag'], rr_ref,
zone_sm.reference))
continue
auto_ptr_privilege_flag = True
# Create a new update group if zone has not been seen before.
try:
update_group, zone_ttl = ug_dict.get(zone_sm)
except (ValueError, TypeError):
# Obtain reverse zone_ttl so PTR rrs can be created
# Use candidate ZI as it always is available.
# zi is published zi
zi = self._get_zi(zone_sm.zi_candidate_id)
if not zi:
log_error("Zone '%s': does not have candidate zi." %
zone_sm.name)
continue
zone_ttl = zi.zone_ttl
update_group = new_update_group(db_session,
None,
zone_sm,
None,
ptr_only=True,
sectag=self.sectag.sectag)
ug_dict[zone_sm] = (update_group, zone_ttl)
# Allocate RR_PTR update record
rr = RR_PTR(label=label,
zone_ttl=zone_ttl,
rdata=ptr_data['hostname'],
disable=ptr_data['disable'],
domain=zone_sm.name,
update_op=update_op)
rr.ref_id = rr_ref.id_ if rr_ref else None
rr.reference = rr_ref
# Chain on RR_PTR update record
update_group.update_ops.append(rr)
# Flush everything to disk
db_session.flush()
# Issue zone refreshes to implement PTR changes
for zone_sm in ug_dict:
if zone_sm.is_disabled():
continue
exec_zonesm(zone_sm, ZoneSMDoRefresh)
# Make sure everything is committed
db_session.commit()
def _data_to_update(self,
name,
update_data,
update_type,
change_by,
admin_privilege=False,
helpdesk_privilege=False):
"""
Construct an update group for a zone, from supplied RRS and comments.
Functional equivalent of _data_to_zi() above, but for incremental
updates
"""
# Function start
db_session = self.db_session
# Check that update_type is supplied
if not update_type:
raise UpdateTypeRequired(name)
# Get zone_sm to get zone ID etc
zone_sm = self._get_zone_sm(name)
zone_id = zone_sm.id_
# See if incremental updates are enabled for zone before queuing any
if not zone_sm.inc_updates:
raise IncrementalUpdatesDisabled(name)
# Don't queue updates for a disabled zone
if zone_sm.is_disabled():
raise ZoneDisabled(name)
# Privilege check for no apex zones - admin only
if not zone_sm.use_apex_ns and not admin_privilege:
raise ZoneAdminPrivilegeNeeded(name)
# Use candidate ZI as it always is available. zi is published zi
zi = self._get_zi(zone_sm.zi_candidate_id)
if not zi:
raise ZiNotFound(name, zone_sm.zi_candidate_id)
# Get value of zone_ttl so that RRs can be created
zone_ttl = zi.zone_ttl
# Create RRs list from published ZI
pzi = PseudoZi(db_session, zi)
# initialise data and zone consistency checking
zi_cname_flag = False
if len([r for r in pzi.rrs if r.type_ == RRTYPE_CNAME]):
zi_cname_flag = True
data_tools = DataTools(db_session, zone_sm, zi_cname_flag)
# Create comments, and set up comment IDs, and stuff for handlng
# RR Groups zi_data structures
data_tools.rr_data_create_comments(update_data,
zone_ttl,
creating_real_zi=False)
try:
# Create new update_group
update_group = new_update_group(db_session, update_type, zone_sm,
change_by)
except IntegrityError as exc:
raise UpdateTypeAlreadyQueued(name, update_type)
# Add RRs to DB and operate on Pseudo ZI
data_tools.add_rrs(lambda: pzi.rrs,
pzi.trial_op_rr,
admin_privilege,
helpdesk_privilege,
update_group=update_group)
data_tools.check_zi_consistency(pzi.rrs)
# Get all data out to DB, and ids etc established.
db_session.flush()
# Refresh zone to implement updates
exec_zonesm(zone_sm, ZoneSMDoRefresh)
# Return auto update info
return data_tools.get_auto_ptr_data()