Example #1
0
 def vacuum_zones(self, age_days=None):
     """
     Destroy zones older than age_days
     """
     self._begin_op()
     db_session = self.db_session
     db_query_slice = get_numeric_setting('db_query_slice', int)
     age_days_from_config = float(zone_cfg.get_row_exc(db_session,
                                 key='zone_del_age'))
     if age_days_from_config <= 0 and age_days is None:
         age_days = get_numeric_setting('zone_del_off_age', float)
     elif age_days is None:
         age_days = age_days_from_config
     age_days = timedelta(days=age_days)
     count = 0
     # Clear old and nuked zones one by one
     id_query = db_session.query(ZoneSM.id_)\
             .filter(ZoneSM.state == ZSTATE_DELETED)\
             .filter(or_(ZoneSM.deleted_start == None,
                         (func.now() - ZoneSM.deleted_start) > age_days))\
             .filter(ZoneSM.zone_files == False)\
             .yield_per(db_query_slice)
     id_results = []
     for zone_id, in id_query:
         id_results.append(zone_id)
     for zone_id in id_results:
         try:
             zone_sm = db_session.query(ZoneSM)\
                     .filter(ZoneSM.id_ == zone_id).one()
         except NoResultFound:
             continue
         if zone_sm.state != ZSTATE_DELETED:
             # Skip this if a customer has undeleted zone in the mean time..
             continue
         db_session.delete(zone_sm)
         db_session.commit()
         count += 1
                 
     # Finally do zone_sm destroy operation to 
     query = db_session.query(ZoneSM)\
             .filter(ZoneSM.state == ZSTATE_DELETED)\
             .filter(or_(ZoneSM.deleted_start == None,
                 (func.now() - ZoneSM.deleted_start) > age_days))
     for zone_sm in query:
         if zone_sm.state != ZSTATE_DELETED:
             # Skip this if a customer has undeleted zone in the mean time..
             continue
         try:
             exec_zonesm(zone_sm, ZoneSMDoDestroy)
         except ZoneSmFailure:
             continue
         count += 1
     result = {'num_deleted': count}
     self._finish_op()
     return result
Example #2
0
 def reset_all_zones(self):
     """
     Reset all zones
     """
     self._begin_op()
     db_session = self.db_session
     id_query = db_session.query(ZoneSM.id_, ZoneSM.name)
     id_query = ZoneSM.query_is_not_disabled_deleted(id_query)
     id_result = id_query.all()
     for zone_id, zone_name in id_result:
         try:
             zone_sm = db_session.query(ZoneSM)\
                     .filter(ZoneSM.id_ == zone_id).one()
         except NoResultFound:
             raise ZoneNotFoundByZoneId(zone_id)
         exec_zonesm(zone_sm, ZoneSMDoReset)
     self._finish_op()
Example #3
0
 def refresh_sg(self, sg_name):
     """
     Refresh all zones on an SG
     """
     self._begin_op()
     db_session = self.db_session
     sg = self._find_sg_byname(sg_name)
     id_query = db_session.query(ZoneSM.id_, ZoneSM.name)\
             .filter(ZoneSM.sg_id == sg.id_) 
     id_query = ZoneSM.query_is_configured(id_query)
     id_result = id_query.all()
     for zone_id, zone_name in id_result:
         try:
             zone_sm = db_session.query(ZoneSM)\
                     .filter(ZoneSM.id_ == zone_id).one()
         except NoResultFound:
             raise ZoneNotFoundByZoneId(zone_id)
         exec_zonesm(zone_sm, ZoneSMDoRefresh)
     self._finish_op()
Example #4
0
    def _queue_auto_ptr_data(self, auto_ptr_data):
        """
        Queue auto PTR data as incremental updates against respective reverse
        zones.
        """
        if not auto_ptr_data:
            return
        if not len(auto_ptr_data):
            return
        if not settings['auto_reverse']:
            return
        db_session = self.db_session
        # Create new update_group
        ug_dict = {}
        auto_ptr_privilege_flag = False
        for ptr_data in auto_ptr_data:
            # Ignore addresses we don't have reverse zone for
            query = db_session.query(ZoneSM)\
                    .join(ReverseNetwork)\
                    .filter(":address  <<= reverse_networks.network")\
                    .params(address = ptr_data['address'])
            query = ZoneSM.query_is_not_deleted(query)
            query = ZoneSM.query_inc_updates(query)
            query = query.order_by(ReverseNetwork.network.desc())\
                    .limit(1)
            try:
                zone_sm = query.one()
            except NoResultFound:
                continue

            # Ignore invalid host names
            if not is_inet_hostname(ptr_data['hostname'], absolute=True,
                                    wildcard=False):
                log_error("Hostname '%s' is not a valid hostname." 
                        % ptr_data['hostname'])
                continue

            # Determine proposed update operation
            update_op = RROP_PTR_UPDATE_FORCE if ptr_data['force_reverse'] \
                            else RROP_PTR_UPDATE

            # Execute privilege checks ahead of time to save unnecessary churn
            # Better than needlessly going through whole rigamorole of 
            # incremental update processing later on for no effect
            #1 See if old PTR exists to retrieve any RR reference
            # Both following also used lower down when generating RR_PTR
            label = label_from_address(ptr_data['address'])
            rr_ref = find_reference(db_session, ptr_data['reference'],
                                        raise_exc=False)
            # query for old record - this generates one select
            # Optimization  - if check has previously suceeded, don't check
            # again as this is all checked further in
            if not auto_ptr_privilege_flag:
                qlabel= label[:label.rfind(zone_sm.name)-1]
                query = db_session.query(ResourceRecord)\
                    .filter(ResourceRecord.label == qlabel)\
                    .filter(ResourceRecord.zi_id == zone_sm.zi_candidate_id)\
                    .filter(ResourceRecord.disable == False)\
                    .filter(ResourceRecord.type_ == RRTYPE_PTR)
                old_rrs = query.all()
                old_rr = old_rrs[0] if len(old_rrs) else None
                
            # Check that we can proceed, only if check has not succeded yet
                if not check_auto_ptr_privilege(rr_ref, self.sectag, zone_sm,
                    old_rr):
                    if old_rr:
                        log_debug("Zone '%s' - can't replace '%s' PTR"
                            " as neither"
                            " sectags '%s' vs '%s'"
                            " references '%s' vs '%s'/'%s' (old PTR/rev zone)"
                            "match ,"
                            " or values not given."
                        % (zone_sm.name, old_rr.label,
                            self.sectag.sectag, settings['admin_sectag'],    
                            rr_ref, old_rr.reference, zone_sm.reference))
                    else:
                        log_debug("Zone '%s' - can't add '%s' PTR as neither"
                                " sectags '%s' vs '%s'"
                                " references '%s' vs '%s' (rev zone) match,"
                                " or values not given."
                            % (zone_sm.name, qlabel,
                                self.sectag.sectag, settings['admin_sectag'],    
                                rr_ref, zone_sm.reference))
                    continue
                auto_ptr_privilege_flag = True

            # Create a new update group if zone has not been seen before.
            try:
                update_group, zone_ttl = ug_dict.get(zone_sm)
            except (ValueError, TypeError):
                # Obtain reverse zone_ttl so PTR rrs can be created
                # Use candidate ZI as it always is available.
                # zi is published zi
                zi = self._get_zi(zone_sm.zi_candidate_id)
                if not zi:
                    log_error("Zone '%s': does not have candidate zi." 
                                % zone_sm.name)
                    continue
                zone_ttl = zi.zone_ttl
                update_group = new_update_group(db_session, None, 
                                        zone_sm, None, ptr_only=True, 
                                        sectag=self.sectag.sectag)
                ug_dict[zone_sm] = (update_group, zone_ttl)
            
            # Allocate RR_PTR update record
            rr = RR_PTR(label=label, zone_ttl=zone_ttl, 
                    rdata=ptr_data['hostname'], disable=ptr_data['disable'],
                    domain=zone_sm.name, update_op=update_op)
            rr.ref_id = rr_ref.id_ if rr_ref else None
            rr.reference = rr_ref

            # Chain on  RR_PTR update record
            update_group.update_ops.append(rr)

        # Flush everything to disk
        db_session.flush()
        # Issue zone refreshes to implement PTR changes
        for zone_sm in ug_dict:
            if zone_sm.is_disabled():
                continue
            exec_zonesm(zone_sm, ZoneSMDoRefresh)
        # Make sure everything is committed
        db_session.commit()
Example #5
0
    def _data_to_update(self, name, update_data, update_type, change_by,
            admin_privilege=False, helpdesk_privilege=False):
        """
        Construct an update group for a zone, from supplied RRS and comments.

        Functional equivalent of _data_to_zi() above, but for incremental 
        updates
        """
        # Function start
        db_session = self.db_session
        # Check that update_type is supplied
        if not update_type:
            raise UpdateTypeRequired(name)
        # Get zone_sm to get zone ID etc
        zone_sm = self._get_zone_sm(name)
        zone_id = zone_sm.id_

        # See if incremental updates are enabled for zone before queuing any
        if not zone_sm.inc_updates:
            raise IncrementalUpdatesDisabled(name)
        # Don't queue updates for a disabled zone
        if zone_sm.is_disabled():
            raise ZoneDisabled(name)
        # Privilege check for no apex zones - admin only 
        if not zone_sm.use_apex_ns and not admin_privilege:
            raise ZoneAdminPrivilegeNeeded(name)
       
        # Use candidate ZI as it always is available.  zi is published zi
        zi = self._get_zi(zone_sm.zi_candidate_id)
        if not zi:
            raise ZiNotFound(name, zone_sm.zi_candidate_id)

        # Get value of zone_ttl so that RRs can be created
        zone_ttl = zi.zone_ttl

        # Create RRs list from published ZI
        pzi = PseudoZi(db_session, zi)

        # initialise data and zone consistency checking
        zi_cname_flag = False
        if len([r for r in pzi.rrs if r.type_ == RRTYPE_CNAME]):
            zi_cname_flag = True
        data_tools = DataTools(db_session, zone_sm, zi_cname_flag)

        # Create comments, and set up comment IDs, and stuff for handlng
        # RR Groups zi_data structures
        data_tools.rr_data_create_comments(update_data, zone_ttl, 
                        creating_real_zi=False)
        try:
            # Create new update_group
            update_group = new_update_group(db_session, update_type, 
                                zone_sm, change_by)
        except IntegrityError as exc:
            raise UpdateTypeAlreadyQueued(name, update_type)

        # Add RRs to DB and operate on Pseudo ZI
        data_tools.add_rrs(lambda :pzi.rrs, pzi.trial_op_rr,
            admin_privilege, helpdesk_privilege, update_group=update_group)

        data_tools.check_zi_consistency(pzi.rrs)

        # Get all data out to DB, and ids etc established.
        db_session.flush()
        # Refresh zone to implement updates
        exec_zonesm(zone_sm, ZoneSMDoRefresh)

        # Return auto update info
        return data_tools.get_auto_ptr_data()
Example #6
0
    def nuke_zones(self, *names, include_deleted=False, toggle_deleted=False, 
            sg_name=None, reference=None):
        """
        Destroy multiple zones.  Multiple names may be given.  Wildcards
        can be used for partial matches.

        This is mainly a command for testing, or cleaning up after a large
        batch zone load goes awry.

        Zones being nuked have their deleted_start set to 1/1/1970, midnight.
        This means they will be immediately reaped by the next vacuum_zones
        command.
        """
        # Deal with SA auto-BEGIN - want fresh transaction to see fresh data
        self._begin_op()
        if not names:
            # No arguments
            self._finish_op()
            raise NoZonesFound('')
       
        db_session = self.db_session
        db_query_slice = get_numeric_setting('db_query_slice', int)
        # We were given some arguments
        zones = []
        # We keep domains and labels in database lowercase
        names = [x.lower() for x in names]
        name_pattern = ' '.join(names)
        names = [x.replace('*', '%') for x in names]
        names = [x.replace('?', '_') for x in names]
        for name in names:
            if not name.endswith('.') and not name.endswith('%'):
                name += '.'
            query = db_session.query(ZoneSM)\
                    .filter(ZoneSM.name.like(name))
            # Don't delete any reverse zones with this command
            query = query.filter(not_(ZoneSM.name.like('%.in-addr.arpa.')))\
                    .filter(not_(ZoneSM.name.like('%.ip6.arpa.')))
            if reference:
                query = query.join(Reference)\
                        .filter(Reference.reference.ilike(reference))
            if sg_name and self.sectag.sectag == settings['admin_sectag']:
                if sg_name not in list_all_sgs(self.db_session):
                    raise NoSgFound(sg_name)
                query = query.join(ServerGroup, ZoneSM.sg_id 
                                    == ServerGroup.id_)\
                        .filter(ServerGroup.name == sg_name)
            if include_deleted:
                pass
            elif toggle_deleted:
                query = query.filter(ZoneSM.state == ZSTATE_DELETED)
            else:
                query = query.filter(ZoneSM.state != ZSTATE_DELETED)
            query = query.yield_per(db_query_slice)
            # The following gives less RAM piggery even though it is slower
            for z in query:
                zones.append(z)
        # Take note of security tags
        if self.sectag.sectag != settings['admin_sectag']:
            zones = [x for x in zones if self.sectag in x.sectags]
        if not zones:
            if len(name_pattern) > 240:
                name_pattern = '* - %s names' % len(names)
            raise NoZonesFound(name_pattern)
        # Mark them all as deleted.
        for zone in zones:
            exec_zonesm(zone, ZoneSMNukeStart)
        self._finish_op()
Example #7
0
    def _queue_auto_ptr_data(self, auto_ptr_data):
        """
        Queue auto PTR data as incremental updates against respective reverse
        zones.
        """
        if not auto_ptr_data:
            return
        if not len(auto_ptr_data):
            return
        if not settings['auto_reverse']:
            return
        db_session = self.db_session
        # Create new update_group
        ug_dict = {}
        auto_ptr_privilege_flag = False
        for ptr_data in auto_ptr_data:
            # Ignore addresses we don't have reverse zone for
            query = db_session.query(ZoneSM)\
                    .join(ReverseNetwork)\
                    .filter(text(":address  <<= reverse_networks.network"))\
                    .params(address = ptr_data['address'])
            query = ZoneSM.query_is_not_deleted(query)
            query = ZoneSM.query_inc_updates(query)
            query = query.order_by(ReverseNetwork.network.desc())\
                    .limit(1)
            try:
                zone_sm = query.one()
            except NoResultFound:
                continue

            # Ignore invalid host names
            if not is_inet_hostname(
                    ptr_data['hostname'], absolute=True, wildcard=False):
                log_error("Hostname '%s' is not a valid hostname." %
                          ptr_data['hostname'])
                continue

            # Determine proposed update operation
            update_op = RROP_PTR_UPDATE_FORCE if ptr_data['force_reverse'] \
                            else RROP_PTR_UPDATE

            # Execute privilege checks ahead of time to save unnecessary churn
            # Better than needlessly going through whole rigamorole of
            # incremental update processing later on for no effect
            #1 See if old PTR exists to retrieve any RR reference
            # Both following also used lower down when generating RR_PTR
            label = label_from_address(ptr_data['address'])
            rr_ref = find_reference(db_session,
                                    ptr_data['reference'],
                                    raise_exc=False)
            # query for old record - this generates one select
            # Optimization  - if check has previously suceeded, don't check
            # again as this is all checked further in
            if not auto_ptr_privilege_flag:
                qlabel = label[:label.rfind(zone_sm.name) - 1]
                query = db_session.query(ResourceRecord)\
                    .filter(ResourceRecord.label == qlabel)\
                    .filter(ResourceRecord.zi_id == zone_sm.zi_candidate_id)\
                    .filter(ResourceRecord.disable == False)\
                    .filter(ResourceRecord.type_ == RRTYPE_PTR)
                old_rrs = query.all()
                old_rr = old_rrs[0] if len(old_rrs) else None

                # Check that we can proceed, only if check has not succeded yet
                if not check_auto_ptr_privilege(rr_ref, self.sectag, zone_sm,
                                                old_rr):
                    if old_rr:
                        log_debug(
                            "Zone '%s' - can't replace '%s' PTR"
                            " as neither"
                            " sectags '%s' vs '%s'"
                            " references '%s' vs '%s'/'%s' (old PTR/rev zone)"
                            "match ,"
                            " or values not given." %
                            (zone_sm.name, old_rr.label, self.sectag.sectag,
                             settings['admin_sectag'], rr_ref,
                             old_rr.reference, zone_sm.reference))
                    else:
                        log_debug("Zone '%s' - can't add '%s' PTR as neither"
                                  " sectags '%s' vs '%s'"
                                  " references '%s' vs '%s' (rev zone) match,"
                                  " or values not given." %
                                  (zone_sm.name, qlabel, self.sectag.sectag,
                                   settings['admin_sectag'], rr_ref,
                                   zone_sm.reference))
                    continue
                auto_ptr_privilege_flag = True

            # Create a new update group if zone has not been seen before.
            try:
                update_group, zone_ttl = ug_dict.get(zone_sm)
            except (ValueError, TypeError):
                # Obtain reverse zone_ttl so PTR rrs can be created
                # Use candidate ZI as it always is available.
                # zi is published zi
                zi = self._get_zi(zone_sm.zi_candidate_id)
                if not zi:
                    log_error("Zone '%s': does not have candidate zi." %
                              zone_sm.name)
                    continue
                zone_ttl = zi.zone_ttl
                update_group = new_update_group(db_session,
                                                None,
                                                zone_sm,
                                                None,
                                                ptr_only=True,
                                                sectag=self.sectag.sectag)
                ug_dict[zone_sm] = (update_group, zone_ttl)

            # Allocate RR_PTR update record
            rr = RR_PTR(label=label,
                        zone_ttl=zone_ttl,
                        rdata=ptr_data['hostname'],
                        disable=ptr_data['disable'],
                        domain=zone_sm.name,
                        update_op=update_op)
            rr.ref_id = rr_ref.id_ if rr_ref else None
            rr.reference = rr_ref

            # Chain on  RR_PTR update record
            update_group.update_ops.append(rr)

        # Flush everything to disk
        db_session.flush()
        # Issue zone refreshes to implement PTR changes
        for zone_sm in ug_dict:
            if zone_sm.is_disabled():
                continue
            exec_zonesm(zone_sm, ZoneSMDoRefresh)
        # Make sure everything is committed
        db_session.commit()
Example #8
0
    def _data_to_update(self,
                        name,
                        update_data,
                        update_type,
                        change_by,
                        admin_privilege=False,
                        helpdesk_privilege=False):
        """
        Construct an update group for a zone, from supplied RRS and comments.

        Functional equivalent of _data_to_zi() above, but for incremental 
        updates
        """
        # Function start
        db_session = self.db_session
        # Check that update_type is supplied
        if not update_type:
            raise UpdateTypeRequired(name)
        # Get zone_sm to get zone ID etc
        zone_sm = self._get_zone_sm(name)
        zone_id = zone_sm.id_

        # See if incremental updates are enabled for zone before queuing any
        if not zone_sm.inc_updates:
            raise IncrementalUpdatesDisabled(name)
        # Don't queue updates for a disabled zone
        if zone_sm.is_disabled():
            raise ZoneDisabled(name)
        # Privilege check for no apex zones - admin only
        if not zone_sm.use_apex_ns and not admin_privilege:
            raise ZoneAdminPrivilegeNeeded(name)

        # Use candidate ZI as it always is available.  zi is published zi
        zi = self._get_zi(zone_sm.zi_candidate_id)
        if not zi:
            raise ZiNotFound(name, zone_sm.zi_candidate_id)

        # Get value of zone_ttl so that RRs can be created
        zone_ttl = zi.zone_ttl

        # Create RRs list from published ZI
        pzi = PseudoZi(db_session, zi)

        # initialise data and zone consistency checking
        zi_cname_flag = False
        if len([r for r in pzi.rrs if r.type_ == RRTYPE_CNAME]):
            zi_cname_flag = True
        data_tools = DataTools(db_session, zone_sm, zi_cname_flag)

        # Create comments, and set up comment IDs, and stuff for handlng
        # RR Groups zi_data structures
        data_tools.rr_data_create_comments(update_data,
                                           zone_ttl,
                                           creating_real_zi=False)
        try:
            # Create new update_group
            update_group = new_update_group(db_session, update_type, zone_sm,
                                            change_by)
        except IntegrityError as exc:
            raise UpdateTypeAlreadyQueued(name, update_type)

        # Add RRs to DB and operate on Pseudo ZI
        data_tools.add_rrs(lambda: pzi.rrs,
                           pzi.trial_op_rr,
                           admin_privilege,
                           helpdesk_privilege,
                           update_group=update_group)

        data_tools.check_zi_consistency(pzi.rrs)

        # Get all data out to DB, and ids etc established.
        db_session.flush()
        # Refresh zone to implement updates
        exec_zonesm(zone_sm, ZoneSMDoRefresh)

        # Return auto update info
        return data_tools.get_auto_ptr_data()