def backup_all(self, bkps_dir): ''' Target: - make a backup of a cluster. Parameters: - bkps_dir: directory where the backup is going to be stored. Return: - a boolean which indicates the success of the process. ''' success = True # Get date and time of the zone init_ts = DateTools.get_date() # Get current year year = str(DateTools.get_year(init_ts)) # Get current month month = str(DateTools.get_month(init_ts)) # Create new directories with the year and the month of the backup bkp_dir = bkps_dir + year + '/' + month + '/' Dir.create_dir(bkp_dir, self.logger) # Set backup's name file_name = self.prefix + 'ht_' + self.connecter.server + \ str(self.connecter.port) + '_cluster_' + init_ts + '.' + \ self.bkp_type # Store the command to do depending on the backup type if self.bkp_type == 'gz': # Zip with gzip command = 'pg_dumpall -U {} -h {} -p {} | gzip > {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, bkp_dir + file_name) elif self.bkp_type == 'bz2': # Zip with bzip2 command = 'pg_dumpall -U {} -h {} -p {} | bzip2 > {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, bkp_dir + file_name) elif self.bkp_type == 'zip': # Zip with zip command = 'pg_dumpall -U {} -h {} -p {} | zip > {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, bkp_dir + file_name) else: # Do not zip command = 'pg_dumpall -U {} -h {} -p {} > {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, bkp_dir + file_name) try: # Execute the command in console result = subprocess.call(command, shell=True) if result != 0: raise Exception() except Exception as e: self.logger.debug('Error en la función "backup_all": {}.'.format( str(e))) success = False return success
def vacuum_dbs(self, vacuum_list): ''' Target: - vacuum a group of PostgreSQL databases. Parameters: - vacuum_list: names of the databases which are going to be vacuumed. ''' if vacuum_list: self.logger.highlight('info', Messenger.BEGINNING_VACUUMER, 'white') for db in vacuum_list: dbname = db['datname'] message = Messenger.PROCESSING_DB.format(dbname=dbname) self.logger.highlight('info', message, 'cyan') # Let the user know whether the database connection is allowed if not db['datallowconn']: message = Messenger.FORBIDDEN_DB_CONNECTION.format( dbname=dbname) self.logger.highlight('warning', message, 'yellow', effect='bold') success = False else: start_time = DateTools.get_current_datetime() # Vacuum the database success = self.vacuum_db(dbname) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if success: message = Messenger.DB_VACUUMER_DONE.format(dbname=dbname, diff=diff) self.logger.highlight('info', message, 'green') else: message = Messenger.DB_VACUUMER_FAIL.format(dbname=dbname) self.logger.highlight('warning', message, 'yellow', effect='bold') self.logger.highlight('info', Messenger.VACUUMER_DONE, 'green', effect='bold')
def alter_dbs_owner(self, alt_list): ''' Target: - change the owner of a group of databases and their tables. Parameters: - alt_list: names of the databases which are going to be altered. ''' self.logger.highlight('info', Msg.PROCESSING_ALTERER, 'white') if alt_list: for db in alt_list: dbname = db['datname'] msg = Msg.PROCESSING_DB.format(dbname=dbname) self.logger.highlight('info', msg, 'cyan') start_time = DateTools.get_current_datetime() # Change the owner of the database success = self.alter_db_owner(db) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if success: msg = Msg.DB_ALTERER_DONE.format(dbname=dbname, diff=diff) self.logger.highlight('info', msg, 'green') else: msg = Msg.DB_ALTERER_FAIL.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow', effect='bold') else: self.logger.highlight('warning', Msg.ALTERER_HAS_NOTHING_TO_DO, 'yellow', effect='bold') self.logger.highlight('info', Msg.ALTERER_DONE, 'green', effect='bold')
def backup_cl(self): ''' Target: - vacuum if necessary and make a backup of a cluster. ''' self.logger.highlight('info', Msg.CHECKING_BACKUP_DIR, 'white') # Create a new directory with the name of the group bkps_dir = self.bkp_path + self.group + Default.CL_BKPS_DIR Dir.create_dir(bkps_dir, self.logger) self.logger.info(Msg.DESTINY_DIR.format(path=bkps_dir)) # Vaccum the databases before the backup process if necessary if self.vacuum: vacuumer = Vacuumer(connecter=self.connecter, logger=self.logger) dbs_all = vacuumer.connecter.get_pg_dbs_data( vacuumer.ex_templates, vacuumer.db_owner) vacuumer.vacuum_dbs(dbs_all) self.logger.highlight('info', Msg.BEGINNING_CL_BACKER, 'white') start_time = DateTools.get_current_datetime() # Make the backup of the cluster success = self.backup_all(bkps_dir) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if success: msg = Msg.CL_BACKER_DONE.format(diff=diff) self.logger.highlight('info', msg, 'green', effect='bold') else: self.logger.highlight('warning', Msg.CL_BACKER_FAIL, 'yellow', effect='bold') self.logger.highlight('info', Msg.BACKER_DONE, 'green', effect='bold')
def backup_cl(self): ''' Target: - vacuum if necessary and make a backup of a cluster. ''' self.logger.highlight('info', Msg.CHECKING_BACKUP_DIR, 'white') # Create a new directory with the name of the group bkps_dir = self.bkp_path + self.group + Default.CL_BKPS_DIR Dir.create_dir(bkps_dir, self.logger) self.logger.info(Msg.DESTINY_DIR.format(path=bkps_dir)) # Vaccum the databases before the backup process if necessary if self.vacuum: vacuumer = Vacuumer(connecter=self.connecter, logger=self.logger) dbs_all = vacuumer.connecter.get_pg_dbs_data(vacuumer.ex_templates, vacuumer.db_owner) vacuumer.vacuum_dbs(dbs_all) self.logger.highlight('info', Msg.BEGINNING_CL_BACKER, 'white') start_time = DateTools.get_current_datetime() # Make the backup of the cluster success = self.backup_all(bkps_dir) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if success: msg = Msg.CL_BACKER_DONE.format(diff=diff) self.logger.highlight('info', msg, 'green', effect='bold') else: self.logger.highlight('warning', Msg.CL_BACKER_FAIL, 'yellow', effect='bold') self.logger.highlight('info', Msg.BACKER_DONE, 'green', effect='bold')
def replicate_pg_db(self): ''' Target: - clone a specified database in PostgreSQL. ''' try: pg_pid = self.connecter.get_pid_str() formatted_sql = Queries.BACKEND_PG_DB_EXISTS.format( pg_pid=pg_pid, target_db=self.original_dbname) self.connecter.cursor.execute(formatted_sql) result = self.connecter.cursor.fetchone() if result: msg = Msg.ACTIVE_CONNS_ERROR.format( dbname=self.original_dbname) self.logger.stop_exe(msg) formatted_query_clone_pg_db = Queries.CLONE_PG_DB.format( dbname=self.new_dbname, original_dbname=self.original_dbname, user=self.connecter.user) msg = Msg.BEGINNING_REPLICATOR.format( original_dbname=self.original_dbname) self.logger.highlight('info', msg, 'white') # Get the database's "datallowconn" value datallowconn = self.connecter.get_datallowconn( self.original_dbname) # If datallowconn is allowed, change it temporarily if datallowconn: # Disallow connections to the database during the # process result = self.connecter.disallow_db_conn(self.original_dbname) if not result: msg = Msg.DISALLOW_CONN_TO_PG_DB_FAIL.format( dbname=self.original_dbname) self.logger.highlight('warning', msg, 'yellow') # self.connecter.cursor.execute('commit') start_time = DateTools.get_current_datetime() # Replicate the database self.connecter.cursor.execute(formatted_query_clone_pg_db) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) # If datallowconn was allowed, leave it as it was if datallowconn: # Allow connections to the database at the end of # the process result = self.connecter.allow_db_conn(self.original_dbname) if not result: msg = Msg.ALLOW_CONN_TO_PG_DB_FAIL.format( dbname=self.original_dbname) self.logger.highlight('warning', msg, 'yellow') msg = Msg.REPLICATE_DB_DONE.format( new_dbname=self.new_dbname, original_dbname=self.original_dbname, diff=diff) self.logger.highlight('info', msg, 'green') self.logger.highlight('info', Msg.REPLICATOR_DONE, 'green', effect='bold') except Exception as e: self.logger.debug('Error en la función "clone_pg_db": ' '{}.'.format(str(e))) self.logger.stop_exe(Msg.REPLICATE_DB_FAIL)
def restore_db_backup(self): ''' Target: - restore a database's backup in PostgreSQL. ''' #replicator = Replicator(self.connecter, self.new_dbname, #Default.RESTORING_TEMPLATE, self.logger) #result = self.connecter.allow_db_conn(Default.RESTORING_TEMPLATE) #if result: #replicator.replicate_pg_db() #self.connecter.disallow_db_conn(Default.RESTORING_TEMPLATE) #else: #self.logger.stop_exe(Messenger.ALLOW_DB_CONN_FAIL.format( #dbname=Default.RESTORING_TEMPLATE)) # Regular expression which must match the backup's name regex = r'.*db_(.+)_(\d{8}_\d{6}_.+)\.(dump|bz2|gz|zip)$' regex = re.compile(regex) if re.match(regex, self.db_backup): # Store the parts of the backup's name (name, date, ext) parts = regex.search(self.db_backup).groups() # Store only the extension to know the type of file ext = parts[2] else: self.logger.stop_exe(Messenger.NO_BACKUP_FORMAT) message = Messenger.BEGINNING_DB_RESTORER.format( db_backup=self.db_backup, new_dbname=self.new_dbname) self.logger.highlight('info', message, 'white') self.logger.info(Messenger.WAIT_PLEASE) if ext == 'gz': command = 'gunzip -c {} -k | pg_restore -U {} -h {} -p {} ' \ '-d {}'.format(self.db_backup, self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname) elif ext == 'bz2': command = 'bunzip2 -c {} -k | pg_restore -U {} -h {} -p {} ' \ '-d {}'.format(self.db_backup, self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname) elif ext == 'zip': command = 'unzip -p {} | pg_restore -U {} -h {} -p {} ' \ '-d {}'.format(self.db_backup, self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname) else: command = 'pg_restore -U {} -h {} -p {} -d {} {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname, self.db_backup) try: start_time = DateTools.get_current_datetime() # Make the restauration of the database result = subprocess.call(command, shell=True) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if result != 0: raise Exception() message = Messenger.RESTORE_DB_DONE.format( db_backup=self.db_backup, new_dbname=self.new_dbname, diff=diff) self.logger.highlight('info', message, 'green') self.logger.highlight('info', Messenger.RESTORER_DONE, 'green', effect='bold') except Exception as e: self.logger.debug('Error en la función "restore_db_backup": ' '{}.'.format(str(e))) message = Messenger.RESTORE_DB_FAIL.format( db_backup=self.db_backup, new_dbname=self.new_dbname) self.logger.stop_exe(message)
def restore_cluster_backup(self): ''' Target: - restore a cluster's backup in PostgreSQL. The cluster must have been created before this process. ''' # Regular expression which must match the backup's name regex = r'.*ht_(.+_cluster)_(\d{8}_\d{6}_.+)\.(dump|bz2|gz|zip)$' regex = re.compile(regex) if re.match(regex, self.cluster_backup): # Store the parts of the backup's name (servername, date, ext) parts = regex.search(self.cluster_backup).groups() # Store only the extension to know the type of file ext = parts[2] else: Messenger.NO_BACKUP_FORMAT message = Messenger.BEGINNING_CL_RESTORER.format( cluster_backup=self.cluster_backup) self.logger.highlight('info', message, 'white') self.logger.info(Messenger.WAIT_PLEASE) # TODO: make dissappear every line about the operation shown in console if ext == 'gz': command = 'gunzip -c {} -k | psql postgres -U {} -h {} ' \ '-p {}'.format( self.cluster_backup, self.connecter.user, self.connecter.server, self.connecter.port) elif ext == 'bz2': command = 'bunzip2 -c {} -k | psql postgres -U {} -h {} ' \ '-p {}'.format( self.cluster_backup, self.connecter.user, self.connecter.server, self.connecter.port) elif ext == 'zip': command = 'unzip -p {} | psql postgres -U {} -h {} -p {}'.format( self.cluster_backup, self.connecter.user, self.connecter.server, self.connecter.port) else: command = 'psql postgres -U {} -h {} -p {} < {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, self.cluster_backup) try: start_time = DateTools.get_current_datetime() # Make the restauration of the cluster result = subprocess.call(command, shell=True) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if result != 0: raise Exception() message = Messenger.RESTORE_CL_DONE.format( cluster_backup=self.cluster_backup, diff=diff) self.logger.highlight('info', message, 'green') self.logger.highlight('info', Messenger.RESTORER_DONE, 'green', effect='bold') except Exception as e: self.logger.debug('Error en la función "restore_cluster_backup": ' '{}.'.format(str(e))) message = Messenger.RESTORE_CL_FAIL.format( cluster_backup=self.cluster_backup) self.logger.stop_exe(message)
def drop_pg_db(self, dbname, pg_superuser): ''' Target: - remove a database in PostgreSQL. Parameters: - dbname: the PostgreSQL database's name which is going to be removed. - pg_superuser: a flag which indicates whether the current user is PostgreSQL superuser or not. ''' delete = False try: self.connecter.cursor.execute(Queries.PG_DB_EXISTS, (dbname, )) result = self.connecter.cursor.fetchone() if result: pg_pid = self.connecter.get_pid_str() formatted_sql = Queries.BACKEND_PG_DB_EXISTS.format( pg_pid=pg_pid, target_db=dbname) self.connecter.cursor.execute(formatted_sql) result = self.connecter.cursor.fetchone() # If there are not any connections to the target database... if not result: # Users who are not superusers will only be able to drop # the databases they own if not pg_superuser: self.connecter.cursor.execute(Queries.GET_PG_DB_OWNER, (dbname, )) db = self.connecter.cursor.fetchone() if db['owner'] != self.connecter.user: msg = Msg.DROP_DB_NOT_ALLOWED.format( user=self.connecter.user, dbname=dbname) self.logger.highlight('warning', msg, 'yellow') else: delete = True else: delete = True if delete: # Get the database's "datallowconn" value datallowconn = self.connecter.get_datallowconn(dbname) # If datallowconn is allowed, change it temporarily if datallowconn: # Disallow connections to the database during the # process result = self.connecter.disallow_db_conn(dbname) if not result: msg = Msg.DISALLOW_CONN_TO_PG_DB_FAIL.format( dbname=dbname) self.logger.highlight('warning', msg, 'yellow') fmt_query_drop_db = Queries.DROP_PG_DB.format( dbname=dbname) start_time = DateTools.get_current_datetime() # Drop the database self.connecter.cursor.execute(fmt_query_drop_db) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) msg = Msg.DROP_DB_DONE.format(dbname=dbname, diff=diff) self.logger.highlight('info', msg, 'green') # If datallowconn was allowed, leave it as it was if datallowconn: # Allow connections to the database at the end of # the process result = self.connecter.allow_db_conn(dbname) if not result: msg = Msg.ALLOW_CONN_TO_PG_DB_FAIL.format( dbname=dbname) self.logger.highlight('warning', msg, 'yellow') else: msg = Msg.ACTIVE_CONNS_ERROR.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow') else: msg = Msg.DB_DOES_NOT_EXIST.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow') except Exception as e: self.logger.debug('Error en la función "drop_pg_db": ' '{}.'.format(str(e))) self.logger.highlight('warning', Msg.DROP_DB_FAIL.format( dbname=dbname), 'yellow')
def restore_db_backup(self): ''' Target: - restore a database's backup in PostgreSQL. ''' replicator = Replicator(self.connecter, self.new_dbname, Default.RESTORING_TEMPLATE, self.logger) result = self.connecter.allow_db_conn(Default.RESTORING_TEMPLATE) if result: replicator.replicate_pg_db() self.connecter.disallow_db_conn(Default.RESTORING_TEMPLATE) else: self.logger.stop_exe( Messenger.ALLOW_DB_CONN_FAIL.format( dbname=Default.RESTORING_TEMPLATE)) # Regular expression which must match the backup's name regex = r'.*db_(.+)_(\d{8}_\d{6}_.+)\.(dump|bz2|gz|zip)$' regex = re.compile(regex) if re.match(regex, self.db_backup): # Store the parts of the backup's name (name, date, ext) parts = regex.search(self.db_backup).groups() # Store only the extension to know the type of file ext = parts[2] else: self.logger.stop_exe(Messenger.NO_BACKUP_FORMAT) message = Messenger.BEGINNING_DB_RESTORER.format( db_backup=self.db_backup, new_dbname=self.new_dbname) self.logger.highlight('info', message, 'white') self.logger.info(Messenger.WAIT_PLEASE) if ext == 'gz': command = 'gunzip -c {} -k | pg_restore -U {} -h {} -p {} ' \ '-d {}'.format(self.db_backup, self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname) elif ext == 'bz2': command = 'bunzip2 -c {} -k | pg_restore -U {} -h {} -p {} ' \ '-d {}'.format(self.db_backup, self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname) elif ext == 'zip': command = 'unzip -p {} | pg_restore -U {} -h {} -p {} ' \ '-d {}'.format(self.db_backup, self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname) else: command = 'pg_restore -U {} -h {} -p {} -d {} {}'.format( self.connecter.user, self.connecter.server, self.connecter.port, self.new_dbname, self.db_backup) try: start_time = DateTools.get_current_datetime() # Make the restauration of the database result = subprocess.call(command, shell=True) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if result != 0: raise Exception() message = Messenger.RESTORE_DB_DONE.format( db_backup=self.db_backup, new_dbname=self.new_dbname, diff=diff) self.logger.highlight('info', message, 'green') self.logger.highlight('info', Messenger.RESTORER_DONE, 'green', effect='bold') except Exception as e: self.logger.debug('Error en la función "restore_db_backup": ' '{}.'.format(str(e))) message = Messenger.RESTORE_DB_FAIL.format( db_backup=self.db_backup, new_dbname=self.new_dbname) self.logger.stop_exe(message)
def drop_pg_db(self, dbname, pg_superuser): ''' Target: - remove a database in PostgreSQL. Parameters: - dbname: the PostgreSQL database's name which is going to be removed. - pg_superuser: a flag which indicates whether the current user is PostgreSQL superuser or not. ''' delete = False try: self.connecter.cursor.execute(Queries.PG_DB_EXISTS, (dbname, )) result = self.connecter.cursor.fetchone() if result: pg_pid = self.connecter.get_pid_str() formatted_sql = Queries.BACKEND_PG_DB_EXISTS.format( pg_pid=pg_pid, target_db=dbname) self.connecter.cursor.execute(formatted_sql) result = self.connecter.cursor.fetchone() # If there are not any connections to the target database... if not result: # Users who are not superusers will only be able to drop # the databases they own if not pg_superuser: self.connecter.cursor.execute(Queries.GET_PG_DB_OWNER, (dbname, )) db = self.connecter.cursor.fetchone() if db['owner'] != self.connecter.user: msg = Msg.DROP_DB_NOT_ALLOWED.format( user=self.connecter.user, dbname=dbname) self.logger.highlight('warning', msg, 'yellow') else: delete = True else: delete = True if delete: # Get the database's "datallowconn" value datallowconn = self.connecter.get_datallowconn(dbname) # If datallowconn is allowed, change it temporarily if datallowconn: # Disallow connections to the database during the # process result = self.connecter.disallow_db_conn(dbname) if not result: msg = Msg.DISALLOW_CONN_TO_PG_DB_FAIL.format( dbname=dbname) self.logger.highlight('warning', msg, 'yellow') fmt_query_drop_db = Queries.DROP_PG_DB.format( dbname=dbname) start_time = DateTools.get_current_datetime() # Drop the database self.connecter.cursor.execute(fmt_query_drop_db) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes( start_time, end_time) msg = Msg.DROP_DB_DONE.format(dbname=dbname, diff=diff) self.logger.highlight('info', msg, 'green') # If datallowconn was allowed, leave it as it was if datallowconn: # Allow connections to the database at the end of # the process result = self.connecter.allow_db_conn(dbname) if not result: msg = Msg.ALLOW_CONN_TO_PG_DB_FAIL.format( dbname=dbname) self.logger.highlight('warning', msg, 'yellow') else: msg = Msg.ACTIVE_CONNS_ERROR.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow') else: msg = Msg.DB_DOES_NOT_EXIST.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow') except Exception as e: self.logger.debug('Error en la función "drop_pg_db": ' '{}.'.format(str(e))) self.logger.highlight('warning', Msg.DROP_DB_FAIL.format(dbname=dbname), 'yellow')
def backup_dbs(self, dbs_all): ''' Target: - make a backup of some specified databases. Parameters: - dbs_all: names of the databases which are going to be backuped. ''' self.logger.highlight('info', Msg.CHECKING_BACKUP_DIR, 'white') # Create a new directory with the name of the group bkps_dir = self.bkp_path + self.group + Default.DB_BKPS_DIR Dir.create_dir(bkps_dir, self.logger) self.logger.info(Msg.DESTINY_DIR.format(path=bkps_dir)) self.logger.highlight('info', Msg.PROCESSING_DB_BACKER, 'white') if dbs_all: for db in dbs_all: dbname = db['datname'] msg = Msg.PROCESSING_DB.format(dbname=dbname) self.logger.highlight('info', msg, 'cyan') # Let the user know whether the database connection is allowed if not db['datallowconn']: msg = Msg.FORBIDDEN_DB_CONNECTION.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow', effect='bold') success = False else: # Vaccum the database before the backup process if # necessary if self.vacuum: self.logger.info( Msg.PRE_VACUUMING_DB.format(dbname=dbname)) vacuumer = Vacuumer(self.connecter, self.in_dbs, self.in_regex, self.in_priority, self.ex_dbs, self.ex_regex, self.ex_templates, self.db_owner, self.logger) # Vacuum the database success = vacuumer.vacuum_db(dbname) if success: msg = Msg.PRE_VACUUMING_DB_DONE.format( dbname=dbname) self.logger.info(msg) else: msg = Msg.PRE_VACUUMING_DB_FAIL.format( dbname=dbname) self.logger.highlight('warning', msg, 'yellow') self.logger.info( Msg.BEGINNING_DB_BACKER.format(dbname=dbname)) start_time = DateTools.get_current_datetime() # Make the backup of the database success = self.backup_db(dbname, bkps_dir) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if success: msg = Msg.DB_BACKER_DONE.format(dbname=dbname, diff=diff) self.logger.highlight('info', msg, 'green') else: msg = Msg.DB_BACKER_FAIL.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow', effect='bold') else: self.logger.highlight('warning', Msg.BACKER_HAS_NOTHING_TO_DO, 'yellow', effect='bold') self.logger.highlight('info', Msg.BACKER_DONE, 'green', effect='bold')
def trim_cluster(self, ht_bkps_list): ''' Target: - remove (if necessary) some cluster's backups, taking into account some parameters in the following order: minimum number of backups to keep > obsolete backups. Parameters: - ht_bkps_list: list of backups of a cluster to analyse and trim. ''' if self.exp_days == -1: # No expiration date x_days_ago = None else: x_days_ago = time.time() - (60 * 60 * 24 * self.exp_days) # Store the total number of backups of the cluster num_bkps = len(ht_bkps_list) # Clone the list to avoid conflict errors when removing ht_bkps_lt = ht_bkps_list[:] unlinked = False self.logger.highlight('info', Messenger.BEGINNING_CL_TRIMMER, 'white') start_time = DateTools.get_current_datetime() for f in ht_bkps_list: # Break if number of backups do not exceed the minimum if num_bkps <= self.min_n_bkps: break file_info = os.stat(f) # Obsolete backup if x_days_ago and file_info.st_ctime < x_days_ago: self.logger.info(Messenger.DELETING_OBSOLETE_BACKUP % f) os.unlink(f) # Remove backup's file unlinked = True # Update the number of backups of the database num_bkps -= 1 ht_bkps_lt.remove(f) # Update the list of cluster's backups end_time = DateTools.get_current_datetime() # Get total size of the backups in Bytes tsize = Dir.get_files_tsize(ht_bkps_lt) # Get total size of the backups in the selected unit of measure tsize_unit = ceil(tsize / self.equivalence) ## UNCOMMENT NEXT SECTION TO PROCEDURE WITH THE BACKUP'S DELETION IF ## THEIR TOTAL SIZE EXCEEDS THE SPECIFIED MAXIMUM SIZE #ht_bkps_list = ht_bkps_lt[:] #for f in ht_bkps_list: ## If there are less backups than the minimum required... #if num_bkps <= self.min_n_bkps: #break #if tsize <= self.max_size_bytes: #break #else: #file_info = os.stat(f) #self.logger.info('Tamaño de copias de seguridad en disco ' #'mayor que {} {}: eliminando el archivo ' #'{}...' % (self.max_size['size'], #self.max_size['unit'], f)) #os.unlink(f) # Remove backup's file #unlinked = True ## Update the number of backups of the cluster #num_bkps -= 1 ## ht_bkps_lt.remove(f) # Update the list of cluster's backups #tsize -= file_info.st_size # Update total size after deletion if not unlinked: message = Messenger.NO_CL_BACKUP_DELETED self.logger.highlight('warning', message, 'yellow') if tsize > self.max_size_bytes: # Total size exceeds the maximum message = Messenger.CL_BKPS_SIZE_EXCEEDED.format( tsize_unit=tsize_unit, size=self.max_size['size'], unit=self.max_size['unit']) self.logger.highlight('warning', message, 'yellow', effect='bold') # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) self.logger.highlight('info', Messenger.CL_TRIMMER_DONE.format( diff=diff), 'green')
def __init__(self, log_dir=None, level=None, mute=False, police=0): ''' Target: - create a logger to store the activity of the program. ''' if log_dir: self.log_dir = log_dir else: # Get script's directory script_dir = os.path.dirname(os.path.realpath(__file__)) # Get program's main directory script_pardir = os.path.abspath(os.path.join(script_dir, os.pardir)) # Get directory to store log files self.log_dir = os.path.join(script_pardir, 'log/') if level in Default.LOG_LEVELS: self.level = level else: self.level = Default.LOG_LEVEL if isinstance(mute, bool): self.mute = mute elif Checker.str_is_bool(mute): self.mute = Casting.str_to_bool(mute) else: self.mute = Default.MUTE # Get current date and time of the zone init_ts = DateTools.get_date() # Get file's name script_filename = os.path.basename(sys.argv[0]) # Get main program's name script_name = os.path.splitext(script_filename)[0] # Set format for the log files' names log_name = script_name + '_' + init_ts + '.log' # Set absolute path for log files self.log_file = os.path.join(self.log_dir, log_name) # Create logger with the main program's name self.logger = logging.getLogger(script_name) # Set DEBUG as maximum verbosity level self.logger.setLevel(logging.DEBUG) # Create a handler for console ch = logging.StreamHandler() # Set the verbosity of console handler to "INFO" ch.setLevel(logging.INFO) # Create a format for console and file logs formatter = logging.Formatter('%(asctime)s - PID %(process)d - ' '%(levelname)-4s - %(message)s', datefmt='%Y.%m.%d_%H:%M:%S_%Z') if self.mute is False: # If log files are required... # Create directory for log files if it does not exist yet if not os.path.exists(self.log_dir): os.makedirs(self.log_dir) # Set size for each log file max_bytes = 4 * 1024 * 1024 # 4MiB # Create a file handler fh = CustomRotatingFileHandler(self.log_file, 'a', max_bytes, 10) # Set the verbosity of console handler to the selected level if self.level == 'debug': fh.setLevel(logging.DEBUG) if self.level == 'info': fh.setLevel(logging.INFO) if self.level == 'warning': fh.setLevel(logging.WARNING) if self.level == 'error': fh.setLevel(logging.ERROR) if self.level == 'critical': fh.setLevel(logging.CRITICAL) fh.setFormatter(formatter) # Set format for file handler ch.setFormatter(formatter) # Set format for console handler self.logger.addHandler(ch) # Add console handler if self.mute is False: # If log files are required... self.logger.addHandler(fh) # Add file handler self.police = police
def send_mail(self, detected_level): ''' Target: - send an email to the specified email addresses. ''' message = Messenger.BEGINNING_MAILER self.logger.highlight('info', message, 'white') # Get current date date = DateTools.get_date(fmt='%d-%m-%Y') time = DateTools.get_date(fmt='%H:%M:%S') zone = DateTools.get_date(fmt='%Z') # Get server name and IP addresses data server = IpAddress.get_hostname(self.logger) internal_ips = '' netifaces = IpAddress.get_netifaces_ips(self.logger) if netifaces: last_index = len(netifaces) - 1 for index, netiface in enumerate(netifaces): internal_ips += '{} > {}'.format(netiface['netiface'], netiface['ip']) if index != last_index: internal_ips += ', ' # Email full info template, for: John Doe <*****@*****.**> ADDR_TMPLT = '{} <{}>' # Sender and recipients email addresses (needed for sending the email) from_email_str = self.from_info['email'] to_emails_list = [dict['email'] for dict in self.to_infos] cc_emails_list = [dict['email'] for dict in self.cc_infos] bcc_emails_list = [dict['email'] for dict in self.bcc_infos] all_emails_list = to_emails_list + cc_emails_list + bcc_emails_list # Sender and recipients full info (used in email message header) from_info_str = ADDR_TMPLT.format(self.from_info['name'], self.from_info['email']) to_infos_str = ', '.join(ADDR_TMPLT.format( dict['name'], dict['email']) for dict in self.to_infos) cc_infos_str = ', '.join(ADDR_TMPLT.format( dict['name'], dict['email']) for dict in self.cc_infos) # Specifying an alternative mail in case the receiver does not have a # mail server with HTML html = self.OP_RESULTS[detected_level].format( op_type=self.OP_TYPES[self.op_type], server_tag=self.server_tag, date=date, time=time, zone=zone, server=server, internal_ips=internal_ips, external_ip=self.external_ip, group=self.group, bkp_path=self.bkp_path, log_file=str(self.logger.log_file)) text = self.OP_RESULTS_NO_HTML[detected_level].format( op_type=self.OP_TYPES[self.op_type], server_tag=self.server_tag, date=date, time=time, zone=zone, server=server, internal_ips=internal_ips, external_ip=self.external_ip, group=self.group, bkp_path=self.bkp_path, log_file=str(self.logger.log_file)) # Specifying other email data (used in email message header) mail = MIMEMultipart('alternative') mail['From'] = from_info_str mail['To'] = to_infos_str mail['Cc'] = cc_infos_str mail['Subject'] = '[INFO] {op_type} results'.format( op_type=self.OP_TYPES[self.op_type].upper()) # Record the MIME types of both parts - text/plain and text/html. part1 = MIMEText(text, 'plain') part2 = MIMEText(html, 'html') # Attach parts into message container. According to RFC 2046, the last # part of a multipart message, in this case the HTML message, is best # and preferred. mail.attach(part1) mail.attach(part2) msg_full = mail.as_string().encode() if all_emails_list: for email in all_emails_list: self.logger.info(Messenger.MAIL_DESTINY.format(email=email)) # Sending the mail try: server = smtplib.SMTP('smtp.gmail.com:587') server.starttls() server.login(self.from_info['email'], self.from_info['pwd']) server.sendmail(from_email_str, all_emails_list, msg_full) server.quit() except smtplib.SMTPException as e: message = Messenger.SEND_MAIL_FAIL self.logger.highlight('info', message, 'yellow') self.logger.debug('Error en la función "send_mail": ' '{}'.format(str(e))) else: message = Messenger.MAILER_HAS_NOTHING_TO_DO self.logger.highlight('info', message, 'yellow') message = Messenger.SEND_MAIL_DONE self.logger.highlight('info', message, 'green')
def backup_dbs(self, dbs_all): ''' Target: - make a backup of some specified databases. Parameters: - dbs_all: names of the databases which are going to be backuped. ''' self.logger.highlight('info', Msg.CHECKING_BACKUP_DIR, 'white') # Create a new directory with the name of the group bkps_dir = self.bkp_path + self.group + Default.DB_BKPS_DIR Dir.create_dir(bkps_dir, self.logger) self.logger.info(Msg.DESTINY_DIR.format(path=bkps_dir)) self.logger.highlight('info', Msg.PROCESSING_DB_BACKER, 'white') if dbs_all: for db in dbs_all: dbname = db['datname'] msg = Msg.PROCESSING_DB.format(dbname=dbname) self.logger.highlight('info', msg, 'cyan') # Let the user know whether the database connection is allowed if not db['datallowconn']: msg = Msg.FORBIDDEN_DB_CONNECTION.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow', effect='bold') success = False else: # Vaccum the database before the backup process if # necessary if self.vacuum: self.logger.info(Msg.PRE_VACUUMING_DB.format( dbname=dbname)) vacuumer = Vacuumer(self.connecter, self.in_dbs, self.in_regex, self.in_priority, self.ex_dbs, self.ex_regex, self.ex_templates, self.db_owner, self.logger) # Vacuum the database success = vacuumer.vacuum_db(dbname) if success: msg = Msg.PRE_VACUUMING_DB_DONE.format( dbname=dbname) self.logger.info(msg) else: msg = Msg.PRE_VACUUMING_DB_FAIL.format( dbname=dbname) self.logger.highlight('warning', msg, 'yellow') self.logger.info(Msg.BEGINNING_DB_BACKER.format( dbname=dbname)) start_time = DateTools.get_current_datetime() # Make the backup of the database success = self.backup_db(dbname, bkps_dir) end_time = DateTools.get_current_datetime() # Get and show the process' duration diff = DateTools.get_diff_datetimes(start_time, end_time) if success: msg = Msg.DB_BACKER_DONE.format(dbname=dbname, diff=diff) self.logger.highlight('info', msg, 'green') else: msg = Msg.DB_BACKER_FAIL.format(dbname=dbname) self.logger.highlight('warning', msg, 'yellow', effect='bold') else: self.logger.highlight('warning', Msg.BACKER_HAS_NOTHING_TO_DO, 'yellow', effect='bold') self.logger.highlight('info', Msg.BACKER_DONE, 'green', effect='bold')