def hard_remove_volume_backup(self, backup_object): try: project_id = backup_object.project_id if project_id not in self.project_list: backup_object.delete_backup() self.openstacksdk.set_project(self.project_list[project_id]) backup = self.openstacksdk.get_backup( uuid=backup_object.backup_id, project_id=project_id ) if backup is None: LOG.info( _( "Backup %s is not existing in Openstack." "Or cinder-backup is not existing in the cloud." % backup_object.backup_id ) ) return backup_object.delete_backup() self.openstacksdk.delete_backup(uuid=backup_object.backup_id) backup_object.delete_backup() except OpenstackSDKException as e: LOG.info( _( "Backup %s deletion failed. Need to delete manually." "%s" % (backup_object.backup_id, str(e)) ) ) # TODO(Alex): Add it into the notification queue # remove from the backup table backup_object.delete_backup()
def create_volume_backup(self, queue): """Initiate the backup of the volume :params: queue: Provide the map of the volume that needs backup. This function will call the backupup api and change the backup_status and backup_id in the queue table. """ project_id = queue.project_id if queue.backup_id == "NULL": try: # NOTE(Alex): no need to wait because we have a cycle time out if project_id not in self.project_list: LOG.warn( _("Project ID %s is not existing in project list" % project_id) ) self.process_non_existing_backup(queue) return self.openstacksdk.set_project(self.project_list[project_id]) LOG.info( _( "Backup for volume %s creating in project %s" % (queue.volume_id, project_id) ) ) volume_backup = self.openstacksdk.create_backup( volume_id=queue.volume_id, project_id=project_id ) queue.backup_id = volume_backup.id queue.backup_status = constants.BACKUP_WIP queue.save() except OpenstackSDKException as error: reason = _( "Backup creation for the volume %s failled. %s" % (queue.volume_id, str(error)) ) LOG.info(reason) self.result.add_failed_backup(project_id, queue.volume_id, reason) parsed = parse.parse("Error in creating volume backup {id}", str(error)) if parsed is not None: queue.backup_id = parsed["id"] queue.backup_status = constants.BACKUP_WIP queue.save() # Added extra exception as OpenstackSDKException does not handle the keystone unauthourized issue. except Exception as error: reason = _( "Backup creation for the volume %s failled. %s" % (queue.volume_id, str(error)) ) LOG.error(reason) self.result.add_failed_backup(project_id, queue.volume_id, reason) parsed = parse.parse("Error in creating volume backup {id}", str(error)) if parsed is not None: queue.backup_id = parsed["id"] queue.backup_status = constants.BACKUP_WIP queue.save() else: # Backup planned task cannot have backup_id in the same cycle. # Remove this task from the task list queue.delete_queue()
def set_project(self, project): LOG.debug(_("Connect as project %s" % project.get("name"))) project_id = project.get("id") if project_id not in self.conn_list: LOG.debug( _("Initiate connection for project %s" % project.get("name"))) conn = self.conn.connect_as_project(project) self.conn_list[project_id] = conn LOG.debug(_("Connect as project %s" % project.get("name"))) self.conn = self.conn_list[project_id]
def _get_ssl_configs(use_ssl): if use_ssl: cert_file = CONF.api.ssl_cert_file key_file = CONF.api.ssl_key_file if cert_file and not os.path.exists(cert_file): raise RuntimeError(_("Unable to find cert_file : %s") % cert_file) if key_file and not os.path.exists(key_file): raise RuntimeError(_("Unable to find key_file : %s") % key_file) return cert_file, key_file else: return None
def _process_todo_tasks(self): LOG.info(_("Creating new backup generators...")) queues_to_start = self.controller.get_queues( filters={"backup_status": constants.BACKUP_PLANNED}) if len(queues_to_start) != 0: for queue in queues_to_start: self.controller.create_volume_backup(queue)
def _backup_cycle_timeout(self): time_delta_dict = xtime.parse_timedelta_string( CONF.conductor.backup_cycle_timout) if time_delta_dict is None: LOG.info( _("Recycle timeout format is invalid. " "Follow <YEARS>y<MONTHS>m<WEEKS>w<DAYS>d<HOURS>h<MINUTES>min<SECONDS>s." )) time_delta_dict = xtime.parse_timedelta_string( constants.DEFAULT_BACKUP_CYCLE_TIMEOUT) rto = xtime.timeago( years=time_delta_dict["years"], months=time_delta_dict["months"], weeks=time_delta_dict["weeks"], days=time_delta_dict["days"], hours=time_delta_dict["hours"], minutes=time_delta_dict["minutes"], seconds=time_delta_dict["seconds"], ) # print(rto.strftime(xtime.DEFAULT_TIME_FORMAT)) # print(self.cycle_start_time) # print(self.cycle_start_time - rto) if rto >= self.cycle_start_time: return True return False
def check_volume_backup_status(self, queue): """Checks the backup status of the volume :params: queue: Provide the map of the volume that needs backup status checked. Call the backups api to see if the backup is successful. """ project_id = queue.project_id # The case in which the error produced before backup gen created. if queue.backup_id == "NULL": self.process_pre_failed_backup(queue) return if project_id not in self.project_list: self.process_non_existing_backup(queue) return self.openstacksdk.set_project(self.project_list[project_id]) backup_gen = self.openstacksdk.get_backup(queue.backup_id) if backup_gen is None: # TODO(Alex): need to check when it is none LOG.info( _("[Beta] Backup status of %s is returning none." % (queue.backup_id)) ) self.process_non_existing_backup(queue) return if backup_gen.status == "error": self.process_failed_backup(queue) elif backup_gen.status == "available": self.process_available_backup(queue) elif backup_gen.status == "creating": LOG.info("Waiting for backup of %s to be completed" % queue.volume_id) else: # "deleting", "restoring", "error_restoring" status self.process_using_backup(queue)
def wrapper(self, *args, **kwargs): try: return func(self, *args, **kwargs) except OpenstackHttpException as ex: if ex.status_code == 403: LOG.warn(_("Token has been expired or rotated!")) self.refresh_openstacksdk() return func(self, *args, **kwargs)
def process_failed_backup(self, task): # 1. notify via email reason = _("The status of backup for the volume %s is error." % task.volume_id) self.result.add_failed_backup(task.project_id, task.volume_id, reason) LOG.warn(reason) # 2. delete backup generator try: self.openstacksdk.delete_backup(uuid=task.backup_id, force=True) except OpenstackHttpException as ex: LOG.error( _( "Failed to delete volume backup %s. %s. Need to delete manually." % (task.backup_id, str(ex)) ) ) # 3. remove failed task from the task queue task.delete_queue()
def add_success_backup(self, project_id, volume_id, backup_id): if project_id not in self.success_backup_list: LOG.error( _("Not registered project is reported for backup result.")) return self.success_backup_list[project_id].append({ "volume_id": volume_id, "backup_id": backup_id, })
def add_failed_backup(self, project_id, volume_id, reason): if project_id not in self.failed_backup_list: LOG.error( _("Not registered project is reported for backup result.")) return self.failed_backup_list[project_id].append({ "volume_id": volume_id, "reason": reason, })
def process_pre_failed_backup(self, task): # 1.notify via email reason = _( "The backup creation for the volume %s was prefailed." % task.volume_id ) self.result.add_failed_backup(task.project_id, task.volume_id, reason) LOG.warn(reason) # 2. remove failed task from the task queue task.delete_queue()
def send_result_email(self): subject = "Backup result" try: if len(CONF.notification.receiver) == 0: return email.send( src_email=CONF.notification.sender_email, src_pwd=CONF.notification.sender_pwd, dest_email=CONF.notification.receiver, subject=subject, content=self.content, smtp_server_domain=CONF.notification.smtp_server_domain, smtp_server_port=CONF.notification.smtp_server_port, ) LOG.info(_("Backup result email sent")) except Exception as e: LOG.error( _("Backup result email send failed. Please check email configuration. %s" % (str(e))))
def _process_wip_tasks(self): LOG.info(_("Processing WIP backup generators...")) # TODO(Alex): Replace this infinite loop with finite time self.cycle_start_time = xtime.get_current_time() # loop - take care of backup result while timeout while 1: queues_started = self.controller.get_queues( filters={"backup_status": constants.BACKUP_WIP}) if len(queues_started) == 0: LOG.info(_("task queue empty")) break if not self._backup_cycle_timeout(): # time in LOG.info(_("cycle timein")) for queue in queues_started: self.controller.check_volume_backup_status(queue) else: # time out LOG.info(_("cycle timeout")) for queue in queues_started: self.controller.hard_cancel_backup_task(queue) break time.sleep(constants.BACKUP_RESULT_CHECK_INTERVAL)
def hard_cancel_backup_task(self, task): try: project_id = task.project_id reason = _("Cancel backup %s because of timeout." % task.backup_id) LOG.info(reason) if project_id not in self.project_list: self.process_non_existing_backup(task) self.openstacksdk.set_project(self.project_list[project_id]) backup = self.openstacksdk.get_backup(task.backup_id) if backup is None: return task.delete_queue() self.openstacksdk.delete_backup(task.backup_id, force=True) task.delete_queue() self.result.add_failed_backup(task.project_id, task.volume_id, reason) except OpenstackSDKException as e: reason = _("Backup %s deletion failed." "%s" % (task.backup_id, str(e))) LOG.info(reason) # remove from the queue table task.delete_queue() self.result.add_failed_backup(task.project_id, task.volume_id, reason)
def soft_remove_backup_task(self, backup_object): try: backup = self.openstacksdk.get_backup(backup_object.backup_id) if backup is None: LOG.info( _( "Backup %s is not existing in Openstack." "Or cinder-backup is not existing in the cloud." % backup_object.backup_id ) ) return backup_object.delete_backup() if backup["status"] in ("available"): self.openstacksdk.delete_backup(backup_object.backup_id) backup_object.delete_backup() elif backup["status"] in ("error", "error_restoring"): # TODO(Alex): need to discuss # now if backup is in error status, then retention service # does not remove it from openstack but removes it from the # backup table so user can delete it on Horizon. backup_object.delete_backup() else: # "deleting", "restoring" LOG.info( _( "Rotation for the backup %s is skipped in this cycle " "because it is in %s status" ) % (backup_object.backup_id, backup["status"]) ) except OpenstackSDKException as e: LOG.info( _("Backup %s deletion failed." "%s" % (backup_object.backup_id, str(e))) ) # TODO(Alex): Add it into the notification queue # remove from the backup table backup_object.delete_backup() return False
def filter_by_volume_status(self, volume_id, project_id): try: volume = self.openstacksdk.get_volume(volume_id, project_id) if volume is None: return False res = volume["status"] in ("available", "in-use") if not res: reason = _( "Volume %s is not backed because it is in %s status" % (volume_id, volume["status"]) ) LOG.info(reason) self.result.add_failed_backup(project_id, volume_id, reason) return res except OpenstackResourceNotFound: return False
def replacement_start_response(status, headers, exc_info=None): """Overrides the default response to make errors parsable.""" try: status_code = int(status.split(" ")[0]) state["status_code"] = status_code except (ValueError, TypeError): # pragma: nocover raise Exception( _("ErrorDocumentMiddleware received an invalid " "status %s") % status) else: if (state["status_code"] // 100) not in (2, 3): # Remove some headers so we can replace them later # when we have the full error message and can # compute the length. headers = [(h, v) for (h, v) in headers if h not in ("Content-Length", "Content-Type")] # Save the headers in case we need to modify them. state["headers"] = headers return start_response(status, headers, exc_info)
def get_threshold_strtime(self): time_delta_dict = xtime.parse_timedelta_string( CONF.conductor.retention_time) if time_delta_dict is None: LOG.info( _("Retention time format is invalid. " "Follow <YEARS>y<MONTHS>m<WEEKS>w<DAYS>d<HOURS>h<MINUTES>min<SECONDS>s." )) return None res = xtime.timeago( years=time_delta_dict["years"], months=time_delta_dict["months"], weeks=time_delta_dict["weeks"], days=time_delta_dict["days"], hours=time_delta_dict["hours"], minutes=time_delta_dict["minutes"], seconds=time_delta_dict["seconds"], ) return res.strftime(xtime.DEFAULT_TIME_FORMAT)
def get_id(source_uuid): """Derive a short (12 character) id from a random UUID. The supplied UUID must be a version 4 UUID object. """ if isinstance(source_uuid, six.string_types): source_uuid = uuid.UUID(source_uuid) if source_uuid.version != 4: raise ValueError(_("Invalid UUID version (%d)") % source_uuid.version) # The "time" field of a v4 UUID contains 60 random bits # (see RFC4122, Section 4.4) random_bytes = _to_byte_string(source_uuid.time, 60) # The first 12 bytes (= 60 bits) of base32-encoded output is our data encoded = base64.b32encode(six.b(random_bytes))[:12] if six.PY3: return encoded.lower().decode("utf-8") else: return encoded.lower()
def check_instance_volumes(self): """Get the list of all the volumes from the project using openstacksdk Function first list all the servers in the project and get the volumes that are attached to the instance. """ queues_map = [] self.refresh_openstacksdk() projects = self.openstacksdk.get_projects() for project in projects: empty_project = True self.project_list[project.id] = project try: servers = self.openstacksdk.get_servers(project_id=project.id) except OpenstackHttpException as ex: LOG.warn( _( "Failed to list servers in project %s. %s" % (project.id, str(ex)) ) ) continue for server in servers: if not self.filter_by_server_metadata(server.metadata): continue if empty_project: empty_project = False self.result.add_project(project.id, project.name) for volume in server.attached_volumes: if not self.filter_by_volume_status(volume["id"], project.id): continue queues_map.append( QueueMapping( project_id=project.id, volume_id=volume["id"], backup_id="NULL", instance_id=server.id, backup_status=constants.BACKUP_PLANNED, ) ) return queues_map
from oslo_config import cfg from staffeln.common import constants from staffeln.i18n import _ conductor_group = cfg.OptGroup( "conductor", title="Conductor Options", help=_("Options under this group are used " "to define Conductor's configuration."), ) backup_opts = [ cfg.IntOpt( "backup_workers", default=1, help=_( "The maximum number of backup processes to " "fork and run. Default to number of CPUs on the host." ), ), cfg.IntOpt( "backup_service_period", default=30, min=10, help=_("The time of bakup period, the unit is one minute."), ), cfg.StrOpt( "backup_cycle_timout", regex=( r"((?P<years>\d+?)y)?((?P<months>\d+?)mon)?((?P<weeks>\d+?)w)?" r"((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)min)?((?P<seconds>\d+?)s)?" ),
from oslo_config import cfg from staffeln.i18n import _ api_group = cfg.OptGroup( "api", title="API options", help=_("Options under this group are used to define staffeln API."), ) connection_opts = [ cfg.StrOpt( "host", default="0.0.0.0", help=_("IP address on which the staffeln API will listen."), ), cfg.PortOpt( "port", default=8808, help=_( "Staffeln API listens on this port number for incoming requests."), ), cfg.BoolOpt("enabled_ssl", default=False, help=_("ssl enabled")), cfg.StrOpt("ssl_key_file", default=False, help=_("ssl key file path")), cfg.StrOpt("ssl_cert_file", default=False, help=_("ssl cert file path")), ] API_OPTS = connection_opts def register_opts(conf): conf.register_group(api_group)
from oslo_config import cfg from oslo_db import options as oslo_db_options from staffeln.conf import paths from staffeln.i18n import _ _DEFAULT_SQL_CONNECTION = "sqlite:///{0}".format( paths.state_path_def("staffeln.sqlite") ) database = cfg.OptGroup( "database", title="Database options", help=_("Options under this group are used for defining database."), ) SQL_OPTS = [ cfg.StrOpt("mysql_engine", default="InnoDB", help=_("MySQL engine to use.")), ] def register_opts(conf): oslo_db_options.set_defaults(conf, connection=_DEFAULT_SQL_CONNECTION) conf.register_group(database) conf.register_opts(SQL_OPTS, group=database) def list_opts(): return [(database, SQL_OPTS)]
import os from oslo_config import cfg from staffeln.i18n import _ PATH_OPTS = [ cfg.StrOpt( "pybasedir", default=os.path.abspath(os.path.join(os.path.dirname(__file__), "../")), help=_("Directory where the staffeln python module is installed."), ), cfg.StrOpt( "bindir", default="$pybasedir/bin", help=_("Directory where staffeln binaries are installed."), ), cfg.StrOpt( "state_path", default="$pybasedir", help=_("Top-level directory for maintaining staffeln's state."), ), ] def basedir_def(*args): """Return an uninterpolated path relative to $pybasedir.""" return os.path.join("$pybasedir", *args) def bindir_def(*args): """Return an uninterpolated path relative to $bindir."""
def _update_task_queue(self): LOG.info(_("Updating backup task queue...")) self.controller.refresh_openstacksdk() self.controller.refresh_backup_result() current_tasks = self.controller.get_queues() self.controller.create_queue(current_tasks)
from oslo_config import cfg from staffeln.i18n import _ notify_group = cfg.OptGroup( "notification", title="Notification options", help=_( "Options under this group are used to define notification settings."), ) email_opts = [ cfg.ListOpt( "receiver", default=[], help=_( "The receivers of the bakcup result by email." "A list of addresses to receive backup result emails to. A bare" " string will be treated as a list with 1 address."), ), cfg.StrOpt( "sender_email", help=_("Log in on an SMTP server that requires authentication." "The user name to authenticate with."), ), # We can remove the sender password as we are using postfix to send mail and we won't be authenticating. cfg.StrOpt( "sender_pwd", help=_("Log in on an SMTP server that requires authentication." "The password for the authentication."), ), cfg.StrOpt(