def vacuum_syslog (self, age_days=None): """ Destroy syslog messages received more than age_days ago """ self._begin_op() db_session = self.db_session if age_days is None: age_days = float(zone_cfg.get_row_exc(db_session, key='syslog_max_age')) age_days = timedelta(days=age_days) count = 0 # Do a straight SQL DELETE first to speed things along # Count events to be deleted syslog_table = sql_data['tables'][SyslogMsg] where_stmt = and_(SyslogMsg.receivedat != None, (func.now() - SyslogMsg.receivedat) > age_days) count_select = select([func.count(syslog_table.c.get('id'))], where_stmt) result = db_session.execute(count_select).fetchall() count += result[0][0] db_session.execute(syslog_table.delete().where(where_stmt)) result = {'num_deleted': count} self._finish_op() return result
def vacuum_event_queue(self, age_days=None): """ Destroy events processed more than age_days ago """ self._begin_op() db_session = self.db_session if age_days is None: age_days = float(zone_cfg.get_row_exc(db_session, key='event_max_age')) age_days = timedelta(days=age_days) count = 0 # Do a straight SQL DELETE first to speed things along # Count events to be deleted event_table = sql_data['tables'][Event] where_stmt = and_(Event.state.in_(event_processed_states), Event.processed != None, (func.now() - Event.processed) > age_days) count_select = select([func.count(event_table.c.get('id'))], where_stmt) result = db_session.execute(count_select).fetchall() count += result[0][0] db_session.execute(event_table.delete().where(where_stmt)) result = {'num_deleted': count} self._finish_op() return result
def set_missing_zi_data(): """ Set missing fields in supplied zi_data to prevent problems """ # Set ZI Zone ttl if not already set if 'zone_ttl' not in zi_data: zi_data['zone_ttl'] = zone_ttl # Set other SOA values in zi_data from defaults # if they are not there. soa_ttl can be None for field in [ 'soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum' ]: if not zi_data.get(field): zi_data[field] = zone_cfg.get_row_exc(db_session, field, sg=zone_sm.sg) # We always update serial number on zone udpdate/publish # but it is nicer and probably less troublesome to replace # an existing serial number that may be out there if not zi_data.get('soa_serial'): if zone_sm.soa_serial: zi_data['soa_serial'] = zone_sm.soa_serial else: # Obviously a new zone zi_data['soa_serial'] = new_zone_soa_serial(db_session)
def get_default_zi_data(db_session, sg_name=None): """ Return default zi data from zone_cfg table This is called from wsgi code or zone_tool """ zi_data = {} soa_fields = ['soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum'] for field in soa_fields: zi_data[field] = zone_cfg.get_row_exc(db_session, field, sg_name=sg_name) zi_data['zone_ttl'] = zone_cfg.get_row_exc(db_session, 'zone_ttl', sg_name=sg_name) zi_data['soa_ttl'] = None zi_data['soa_serial'] = new_zone_soa_serial(db_session) return zi_data
def vacuum_zis(self, age_days=None, zi_max_num=None): """ Age ZIs according to age_days and zi_max_num """ self._begin_op() db_session = self.db_session db_query_slice = get_numeric_setting('db_query_slice', int) if age_days is None: age_days = float(zone_cfg.get_row_exc(db_session, key='zi_max_age')) age_days = timedelta(days=age_days) if zi_max_num is None: zi_max_num = int(zone_cfg.get_row_exc(db_session, key='zi_max_num')) stmt = db_session.query(ZoneInstance.zone_id, func.count(ZoneInstance.id_).label('zi_count'))\ .group_by(ZoneInstance.zone_id).subquery() zone_sm_query = db_session.query(ZoneSM)\ .filter(ZoneSM.state != ZSTATE_DELETED)\ .outerjoin(stmt, ZoneSM.id_ == stmt.c.zone_id)\ .filter(stmt.c.zi_count > zi_max_num)\ .yield_per(db_query_slice) count = 0 for zone_sm in zone_sm_query: zi_keep = db_session.query(ZoneInstance.id_)\ .filter(ZoneInstance.zone_id == zone_sm.id_)\ .order_by(desc(ZoneInstance.mtime))\ .limit(zi_max_num) zi_query = db_session.query(ZoneInstance)\ .filter(ZoneInstance.zone_id == zone_sm.id_)\ .filter(ZoneInstance.id_ != zone_sm.zi_id)\ .filter(not_(ZoneInstance.id_.in_(zi_keep)))\ .filter(ZoneInstance.mtime < (func.now() - age_days)) for zi in zi_query: if (zi.id_ == zone_sm.zi_id or zi.id_ == zone_sm.zi_candidate_id): # Skip if this ZI has ben selected for republishing in # the mean time continue db_session.delete(zi) count += 1 result = {'num_deleted': count} self._finish_op() return result
def vacuum_zones(self, age_days=None): """ Destroy zones older than age_days """ self._begin_op() db_session = self.db_session db_query_slice = get_numeric_setting('db_query_slice', int) age_days_from_config = float(zone_cfg.get_row_exc(db_session, key='zone_del_age')) if age_days_from_config <= 0 and age_days is None: age_days = get_numeric_setting('zone_del_off_age', float) elif age_days is None: age_days = age_days_from_config age_days = timedelta(days=age_days) count = 0 # Clear old and nuked zones one by one id_query = db_session.query(ZoneSM.id_)\ .filter(ZoneSM.state == ZSTATE_DELETED)\ .filter(or_(ZoneSM.deleted_start == None, (func.now() - ZoneSM.deleted_start) > age_days))\ .filter(ZoneSM.zone_files == False)\ .yield_per(db_query_slice) id_results = [] for zone_id, in id_query: id_results.append(zone_id) for zone_id in id_results: try: zone_sm = db_session.query(ZoneSM)\ .filter(ZoneSM.id_ == zone_id).one() except NoResultFound: continue if zone_sm.state != ZSTATE_DELETED: # Skip this if a customer has undeleted zone in the mean time.. continue db_session.delete(zone_sm) db_session.commit() count += 1 # Finally do zone_sm destroy operation to query = db_session.query(ZoneSM)\ .filter(ZoneSM.state == ZSTATE_DELETED)\ .filter(or_(ZoneSM.deleted_start == None, (func.now() - ZoneSM.deleted_start) > age_days)) for zone_sm in query: if zone_sm.state != ZSTATE_DELETED: # Skip this if a customer has undeleted zone in the mean time.. continue try: exec_zonesm(zone_sm, ZoneSMDoDestroy) except ZoneSmFailure: continue count += 1 result = {'num_deleted': count} self._finish_op() return result
def get_default_zi_data(db_session, sg_name=None): """ Return default zi data from zone_cfg table This is called from wsgi code or zone_tool """ zi_data = {} soa_fields = [ 'soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum' ] for field in soa_fields: zi_data[field] = zone_cfg.get_row_exc(db_session, field, sg_name=sg_name) zi_data['zone_ttl'] = zone_cfg.get_row_exc(db_session, 'zone_ttl', sg_name=sg_name) zi_data['soa_ttl'] = None zi_data['soa_serial'] = new_zone_soa_serial(db_session) return zi_data
def move_server_sg(db_session, server_name, sg_name=None): """ Move a server between SGs """ server_sm = find_server_byname(db_session, server_name) if not server_sm.is_disabled(): raise ServerNotDisabled(server_sm.name) if not sg_name: sg_name = zone_cfg.get_row_exc(db_session, 'default_sg') if not sg_name in list_all_sgs(db_session): raise NoSgFound(sg_name) sg = find_sg_byname(db_session, sg_name, raise_exc=True) server_sm.set_sg(sg)
def vacuum_pare_deleted_zone_zis(self, age_days=None): """ Pare ZIs on deleted zones older than age_days """ self._begin_op() db_session = self.db_session db_query_slice = get_numeric_setting('db_query_slice', int) age_days_from_config = float(zone_cfg.get_row_exc(db_session, key='zone_del_pare_age')) if age_days_from_config <= 0 and age_days is None: return {'num_deleted': 0} if age_days is None: age_days = age_days_from_config age_days = timedelta(days=age_days) stmt = db_session.query(ZoneInstance.zone_id, func.count(ZoneInstance.id_).label('zi_count'))\ .group_by(ZoneInstance.zone_id).subquery() zone_sm_query = db_session.query(ZoneSM)\ .filter(ZoneSM.state == ZSTATE_DELETED)\ .outerjoin(stmt, ZoneSM.id_ == stmt.c.zone_id)\ .filter(stmt.c.zi_count > 1)\ .filter(and_(ZoneSM.deleted_start != None, (func.now() - ZoneSM.deleted_start) > age_days))\ .yield_per(db_query_slice) count = 0 for zone_sm in zone_sm_query: zi_query = db_session.query(ZoneInstance)\ .filter(ZoneInstance.zone_id == zone_sm.id_)\ .filter(ZoneInstance.id_ != zone_sm.zi_id) if zone_sm.state != ZSTATE_DELETED: # Skip this if a customer has undeleted zone in the mean time.. continue for zi in zi_query: if (zi.id_ == zone_sm.zi_id or zi.id_ == zone_sm.zi_candidate_id): # Skip if this ZI has published or selected to be published continue db_session.delete(zi) count += 1 result = {'num_deleted': count} self._finish_op() return result
def new_server(db_session, server_name, address, sg_name, server_type=None, ssh_address=None): """ Create a new server """ server_name = server_name.lower() if server_name.endswith('.'): server_name = server_name[:-1] if not sg_name: sg_name = zone_cfg.get_row_exc(db_session, 'default_sg') if not sg_name in list_all_sgs(db_session): raise NoSgFound(sg_name) try: server_list = db_session.query(ServerSM)\ .filter(ServerSM.name == server_name).all() if len(server_list): raise ServerExists(server_name) except NoResultFound: pass if not server_type: server_type = zone_cfg.get_row(db_session, 'default_stype', raise_exc=True) server_sm = ServerSM(server_name, address, sg_name, server_type, ssh_address) try: db_session.add(server_sm) db_session.flush() except IntegrityError as exc: raise ServerAddressExists(address) sg = find_sg_byname(db_session, sg_name, raise_exc=True) server_sm.set_sg(sg) db_session.flush() return server_sm
def set_missing_zi_data(): """ Set missing fields in supplied zi_data to prevent problems """ # Set ZI Zone ttl if not already set if 'zone_ttl' not in zi_data: zi_data['zone_ttl'] = zone_ttl # Set other SOA values in zi_data from defaults # if they are not there. soa_ttl can be None for field in ['soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum']: if not zi_data.get(field): zi_data[field] = zone_cfg.get_row_exc(db_session, field, sg=zone_sm.sg) # We always update serial number on zone udpdate/publish # but it is nicer and probably less troublesome to replace # an existing serial number that may be out there if not zi_data.get('soa_serial'): if zone_sm.soa_serial: zi_data['soa_serial'] = zone_sm.soa_serial else: # Obviously a new zone zi_data['soa_serial'] = new_zone_soa_serial(db_session)
def _data_to_zi(self, name, zi_data, change_by, normalize_ttls=False, admin_privilege=False, helpdesk_privilege=False): """ Construct a new ZI, RRS and comments, from zone_data. """ def set_missing_zi_data(): """ Set missing fields in supplied zi_data to prevent problems """ # Set ZI Zone ttl if not already set if 'zone_ttl' not in zi_data: zi_data['zone_ttl'] = zone_ttl # Set other SOA values in zi_data from defaults # if they are not there. soa_ttl can be None for field in ['soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum']: if not zi_data.get(field): zi_data[field] = zone_cfg.get_row_exc(db_session, field, sg=zone_sm.sg) # We always update serial number on zone udpdate/publish # but it is nicer and probably less troublesome to replace # an existing serial number that may be out there if not zi_data.get('soa_serial'): if zone_sm.soa_serial: zi_data['soa_serial'] = zone_sm.soa_serial else: # Obviously a new zone zi_data['soa_serial'] = new_zone_soa_serial(db_session) def check_zi_data(): """ Check incoming zi_data attributes for correctness """ for field in ['soa_mname', 'soa_rname']: validate_zi_hostname(name, field, zi_data[field]) for field in ['soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum', 'soa_ttl', 'zone_ttl']: if field == 'soa_ttl' and not zi_data.get(field): # SOA TTL can be None continue validate_zi_ttl(name, field, zi_data[field]) for field in ['soa_serial']: if field == 'soa_serial' and zi_data.get(field, None) == None: # SOA serial can be None continue # Check incoming data type of soa_serial if not isinstance(zi_data['soa_serial'], int): raise SOASerialTypeError(name) if not ( 0 < zi_data['soa_serial'] <= (2**32-1)): # RFC 2136 Section 4.2 AO serial cannot be zero raise SOASerialRangeError(name) # Function start db_session = self.db_session # Get zone_sm to get zone ID etc zone_sm = self._get_zone_sm(name) zone_id = zone_sm.id_ # initialise data and zone consistency checking data_tools = DataTools(db_session, zone_sm) # Sort out a candidate value for zone_ttl so that RRs can be created zone_ttl = zi_data.get('zone_ttl', zone_cfg.get_row_exc(db_session, 'zone_ttl', sg=zone_sm.sg)) zone_ttl_supplied = 'zone_ttl' in zi_data # Create comments, and set up comment IDs, and stuff for handlng # RR Groups zi_data structures data_tools.rr_data_create_comments(zi_data, zone_ttl) # Deal with ZI data problems, and supply defaults if missing set_missing_zi_data() check_zi_data() # This constructor call sets attributes in zi as well! zi = ZoneInstance(change_by=change_by, **zi_data) db_session.add(zi) apex_comment = data_tools.get_apex_comment() if apex_comment: zi.add_apex_comment(apex_comment) # Get zi.id_ zi.zone_id from database db_session.flush() # Add RRs to zi # Note use of lambda so that list of rrs is always refreshed in # function data_tools.add_rrs(lambda :zi.rrs, zi.add_rr, admin_privilege, helpdesk_privilege) # tie zi into data_structures zone_sm.all_zis.append(zi) zi.zone = zone_sm db_session.flush() # Normalise TTLs here if normalize_ttls and zone_ttl_supplied: zi.normalize_ttls() # Update SOA and NS records - can't hurt to do it here # This also cleans out any incoming apex NS records if # client should not be setting them. zi.update_apex(db_session) # Update Zone TTLs for clean initialisation zi.update_zone_ttls() db_session.flush() # Check zone consistency. Do this here as Apex RRs need to be complete. data_tools.check_zi_consistency(zi.rrs) return zi, data_tools.get_auto_ptr_data()
def get_config_default(self, config_key): """ Get the default value for a configuration key """ self.refresh_db_session() return zone_cfg.get_row_exc(self.db_session, config_key)
def _data_to_zi(self, name, zi_data, change_by, normalize_ttls=False, admin_privilege=False, helpdesk_privilege=False): """ Construct a new ZI, RRS and comments, from zone_data. """ def set_missing_zi_data(): """ Set missing fields in supplied zi_data to prevent problems """ # Set ZI Zone ttl if not already set if 'zone_ttl' not in zi_data: zi_data['zone_ttl'] = zone_ttl # Set other SOA values in zi_data from defaults # if they are not there. soa_ttl can be None for field in [ 'soa_mname', 'soa_rname', 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum' ]: if not zi_data.get(field): zi_data[field] = zone_cfg.get_row_exc(db_session, field, sg=zone_sm.sg) # We always update serial number on zone udpdate/publish # but it is nicer and probably less troublesome to replace # an existing serial number that may be out there if not zi_data.get('soa_serial'): if zone_sm.soa_serial: zi_data['soa_serial'] = zone_sm.soa_serial else: # Obviously a new zone zi_data['soa_serial'] = new_zone_soa_serial(db_session) def check_zi_data(): """ Check incoming zi_data attributes for correctness """ for field in ['soa_mname', 'soa_rname']: validate_zi_hostname(name, field, zi_data[field]) for field in [ 'soa_refresh', 'soa_retry', 'soa_expire', 'soa_minimum', 'soa_ttl', 'zone_ttl' ]: if field == 'soa_ttl' and not zi_data.get(field): # SOA TTL can be None continue validate_zi_ttl(name, field, zi_data[field]) for field in ['soa_serial']: if field == 'soa_serial' and zi_data.get(field, None) == None: # SOA serial can be None continue # Check incoming data type of soa_serial if not isinstance(zi_data['soa_serial'], int): raise SOASerialTypeError(name) if not (0 < zi_data['soa_serial'] <= (2**32 - 1)): # RFC 2136 Section 4.2 AO serial cannot be zero raise SOASerialRangeError(name) # Function start db_session = self.db_session # Get zone_sm to get zone ID etc zone_sm = self._get_zone_sm(name) zone_id = zone_sm.id_ # initialise data and zone consistency checking data_tools = DataTools(db_session, zone_sm) # Sort out a candidate value for zone_ttl so that RRs can be created zone_ttl = zi_data.get( 'zone_ttl', zone_cfg.get_row_exc(db_session, 'zone_ttl', sg=zone_sm.sg)) zone_ttl_supplied = 'zone_ttl' in zi_data # Create comments, and set up comment IDs, and stuff for handlng # RR Groups zi_data structures data_tools.rr_data_create_comments(zi_data, zone_ttl) # Deal with ZI data problems, and supply defaults if missing set_missing_zi_data() check_zi_data() # This constructor call sets attributes in zi as well! zi = ZoneInstance(change_by=change_by, **zi_data) db_session.add(zi) apex_comment = data_tools.get_apex_comment() if apex_comment: zi.add_apex_comment(apex_comment) # Get zi.id_ zi.zone_id from database db_session.flush() # Add RRs to zi # Note use of lambda so that list of rrs is always refreshed in # function data_tools.add_rrs(lambda: zi.rrs, zi.add_rr, admin_privilege, helpdesk_privilege) # tie zi into data_structures zone_sm.all_zis.append(zi) zi.zone = zone_sm db_session.flush() # Normalise TTLs here if normalize_ttls and zone_ttl_supplied: zi.normalize_ttls() # Update SOA and NS records - can't hurt to do it here # This also cleans out any incoming apex NS records if # client should not be setting them. zi.update_apex(db_session) # Update Zone TTLs for clean initialisation zi.update_zone_ttls() db_session.flush() # Check zone consistency. Do this here as Apex RRs need to be complete. data_tools.check_zi_consistency(zi.rrs) return zi, data_tools.get_auto_ptr_data()