def hard_remove_volume_backup(self, backup_object):
try:
project_id = backup_object.project_id
if project_id not in self.project_list:
backup_object.delete_backup()
self.openstacksdk.set_project(self.project_list[project_id])
backup = self.openstacksdk.get_backup(
uuid=backup_object.backup_id, project_id=project_id
)
if backup is None:
LOG.info(
_(
"Backup %s is not existing in Openstack."
"Or cinder-backup is not existing in the cloud."
% backup_object.backup_id
)
)
return backup_object.delete_backup()
self.openstacksdk.delete_backup(uuid=backup_object.backup_id)
backup_object.delete_backup()
except OpenstackSDKException as e:
LOG.info(
_(
"Backup %s deletion failed. Need to delete manually."
"%s" % (backup_object.backup_id, str(e))
)
)
# TODO(Alex): Add it into the notification queue
# remove from the backup table
backup_object.delete_backup()
def create_volume_backup(self, queue):
"""Initiate the backup of the volume
:params: queue: Provide the map of the volume that needs
backup.
This function will call the backupup api and change the
backup_status and backup_id in the queue table.
"""
project_id = queue.project_id
if queue.backup_id == "NULL":
try:
# NOTE(Alex): no need to wait because we have a cycle time out
if project_id not in self.project_list:
LOG.warn(
_("Project ID %s is not existing in project list" % project_id)
)
self.process_non_existing_backup(queue)
return
self.openstacksdk.set_project(self.project_list[project_id])
LOG.info(
_(
"Backup for volume %s creating in project %s"
% (queue.volume_id, project_id)
)
)
volume_backup = self.openstacksdk.create_backup(
volume_id=queue.volume_id, project_id=project_id
)
queue.backup_id = volume_backup.id
queue.backup_status = constants.BACKUP_WIP
queue.save()
except OpenstackSDKException as error:
reason = _(
"Backup creation for the volume %s failled. %s"
% (queue.volume_id, str(error))
)
LOG.info(reason)
self.result.add_failed_backup(project_id, queue.volume_id, reason)
parsed = parse.parse("Error in creating volume backup {id}", str(error))
if parsed is not None:
queue.backup_id = parsed["id"]
queue.backup_status = constants.BACKUP_WIP
queue.save()
# Added extra exception as OpenstackSDKException does not handle the keystone unauthourized issue.
except Exception as error:
reason = _(
"Backup creation for the volume %s failled. %s"
% (queue.volume_id, str(error))
)
LOG.error(reason)
self.result.add_failed_backup(project_id, queue.volume_id, reason)
parsed = parse.parse("Error in creating volume backup {id}", str(error))
if parsed is not None:
queue.backup_id = parsed["id"]
queue.backup_status = constants.BACKUP_WIP
queue.save()
else:
# Backup planned task cannot have backup_id in the same cycle.
# Remove this task from the task list
queue.delete_queue()
def set_project(self, project):
LOG.debug(_("Connect as project %s" % project.get("name")))
project_id = project.get("id")
if project_id not in self.conn_list:
LOG.debug(
_("Initiate connection for project %s" % project.get("name")))
conn = self.conn.connect_as_project(project)
self.conn_list[project_id] = conn
LOG.debug(_("Connect as project %s" % project.get("name")))
self.conn = self.conn_list[project_id]
def _get_ssl_configs(use_ssl):
if use_ssl:
cert_file = CONF.api.ssl_cert_file
key_file = CONF.api.ssl_key_file
if cert_file and not os.path.exists(cert_file):
raise RuntimeError(_("Unable to find cert_file : %s") % cert_file)
if key_file and not os.path.exists(key_file):
raise RuntimeError(_("Unable to find key_file : %s") % key_file)
return cert_file, key_file
else:
return None
def _process_todo_tasks(self):
LOG.info(_("Creating new backup generators..."))
queues_to_start = self.controller.get_queues(
filters={"backup_status": constants.BACKUP_PLANNED})
if len(queues_to_start) != 0:
for queue in queues_to_start:
self.controller.create_volume_backup(queue)
def _backup_cycle_timeout(self):
time_delta_dict = xtime.parse_timedelta_string(
CONF.conductor.backup_cycle_timout)
if time_delta_dict is None:
LOG.info(
_("Recycle timeout format is invalid. "
"Follow <YEARS>y<MONTHS>m<WEEKS>w<DAYS>d<HOURS>h<MINUTES>min<SECONDS>s."
))
time_delta_dict = xtime.parse_timedelta_string(
constants.DEFAULT_BACKUP_CYCLE_TIMEOUT)
rto = xtime.timeago(
years=time_delta_dict["years"],
months=time_delta_dict["months"],
weeks=time_delta_dict["weeks"],
days=time_delta_dict["days"],
hours=time_delta_dict["hours"],
minutes=time_delta_dict["minutes"],
seconds=time_delta_dict["seconds"],
)
# print(rto.strftime(xtime.DEFAULT_TIME_FORMAT))
# print(self.cycle_start_time)
# print(self.cycle_start_time - rto)
if rto >= self.cycle_start_time:
return True
return False
def check_volume_backup_status(self, queue):
"""Checks the backup status of the volume
:params: queue: Provide the map of the volume that needs backup
status checked.
Call the backups api to see if the backup is successful.
"""
project_id = queue.project_id
# The case in which the error produced before backup gen created.
if queue.backup_id == "NULL":
self.process_pre_failed_backup(queue)
return
if project_id not in self.project_list:
self.process_non_existing_backup(queue)
return
self.openstacksdk.set_project(self.project_list[project_id])
backup_gen = self.openstacksdk.get_backup(queue.backup_id)
if backup_gen is None:
# TODO(Alex): need to check when it is none
LOG.info(
_("[Beta] Backup status of %s is returning none." % (queue.backup_id))
)
self.process_non_existing_backup(queue)
return
if backup_gen.status == "error":
self.process_failed_backup(queue)
elif backup_gen.status == "available":
self.process_available_backup(queue)
elif backup_gen.status == "creating":
LOG.info("Waiting for backup of %s to be completed" % queue.volume_id)
else: # "deleting", "restoring", "error_restoring" status
self.process_using_backup(queue)
def wrapper(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except OpenstackHttpException as ex:
if ex.status_code == 403:
LOG.warn(_("Token has been expired or rotated!"))
self.refresh_openstacksdk()
return func(self, *args, **kwargs)
def process_failed_backup(self, task):
# 1. notify via email
reason = _("The status of backup for the volume %s is error." % task.volume_id)
self.result.add_failed_backup(task.project_id, task.volume_id, reason)
LOG.warn(reason)
# 2. delete backup generator
try:
self.openstacksdk.delete_backup(uuid=task.backup_id, force=True)
except OpenstackHttpException as ex:
LOG.error(
_(
"Failed to delete volume backup %s. %s. Need to delete manually."
% (task.backup_id, str(ex))
)
)
# 3. remove failed task from the task queue
task.delete_queue()
def add_success_backup(self, project_id, volume_id, backup_id):
if project_id not in self.success_backup_list:
LOG.error(
_("Not registered project is reported for backup result."))
return
self.success_backup_list[project_id].append({
"volume_id": volume_id,
"backup_id": backup_id,
})
def add_failed_backup(self, project_id, volume_id, reason):
if project_id not in self.failed_backup_list:
LOG.error(
_("Not registered project is reported for backup result."))
return
self.failed_backup_list[project_id].append({
"volume_id": volume_id,
"reason": reason,
})
def process_pre_failed_backup(self, task):
# 1.notify via email
reason = _(
"The backup creation for the volume %s was prefailed." % task.volume_id
)
self.result.add_failed_backup(task.project_id, task.volume_id, reason)
LOG.warn(reason)
# 2. remove failed task from the task queue
task.delete_queue()
def send_result_email(self):
subject = "Backup result"
try:
if len(CONF.notification.receiver) == 0:
return
email.send(
src_email=CONF.notification.sender_email,
src_pwd=CONF.notification.sender_pwd,
dest_email=CONF.notification.receiver,
subject=subject,
content=self.content,
smtp_server_domain=CONF.notification.smtp_server_domain,
smtp_server_port=CONF.notification.smtp_server_port,
)
LOG.info(_("Backup result email sent"))
except Exception as e:
LOG.error(
_("Backup result email send failed. Please check email configuration. %s"
% (str(e))))
def _process_wip_tasks(self):
LOG.info(_("Processing WIP backup generators..."))
# TODO(Alex): Replace this infinite loop with finite time
self.cycle_start_time = xtime.get_current_time()
# loop - take care of backup result while timeout
while 1:
queues_started = self.controller.get_queues(
filters={"backup_status": constants.BACKUP_WIP})
if len(queues_started) == 0:
LOG.info(_("task queue empty"))
break
if not self._backup_cycle_timeout(): # time in
LOG.info(_("cycle timein"))
for queue in queues_started:
self.controller.check_volume_backup_status(queue)
else: # time out
LOG.info(_("cycle timeout"))
for queue in queues_started:
self.controller.hard_cancel_backup_task(queue)
break
time.sleep(constants.BACKUP_RESULT_CHECK_INTERVAL)
def hard_cancel_backup_task(self, task):
try:
project_id = task.project_id
reason = _("Cancel backup %s because of timeout." % task.backup_id)
LOG.info(reason)
if project_id not in self.project_list:
self.process_non_existing_backup(task)
self.openstacksdk.set_project(self.project_list[project_id])
backup = self.openstacksdk.get_backup(task.backup_id)
if backup is None:
return task.delete_queue()
self.openstacksdk.delete_backup(task.backup_id, force=True)
task.delete_queue()
self.result.add_failed_backup(task.project_id, task.volume_id, reason)
except OpenstackSDKException as e:
reason = _("Backup %s deletion failed." "%s" % (task.backup_id, str(e)))
LOG.info(reason)
# remove from the queue table
task.delete_queue()
self.result.add_failed_backup(task.project_id, task.volume_id, reason)
def soft_remove_backup_task(self, backup_object):
try:
backup = self.openstacksdk.get_backup(backup_object.backup_id)
if backup is None:
LOG.info(
_(
"Backup %s is not existing in Openstack."
"Or cinder-backup is not existing in the cloud."
% backup_object.backup_id
)
)
return backup_object.delete_backup()
if backup["status"] in ("available"):
self.openstacksdk.delete_backup(backup_object.backup_id)
backup_object.delete_backup()
elif backup["status"] in ("error", "error_restoring"):
# TODO(Alex): need to discuss
# now if backup is in error status, then retention service
# does not remove it from openstack but removes it from the
# backup table so user can delete it on Horizon.
backup_object.delete_backup()
else: # "deleting", "restoring"
LOG.info(
_(
"Rotation for the backup %s is skipped in this cycle "
"because it is in %s status"
)
% (backup_object.backup_id, backup["status"])
)
except OpenstackSDKException as e:
LOG.info(
_("Backup %s deletion failed." "%s" % (backup_object.backup_id, str(e)))
)
# TODO(Alex): Add it into the notification queue
# remove from the backup table
backup_object.delete_backup()
return False
def filter_by_volume_status(self, volume_id, project_id):
try:
volume = self.openstacksdk.get_volume(volume_id, project_id)
if volume is None:
return False
res = volume["status"] in ("available", "in-use")
if not res:
reason = _(
"Volume %s is not backed because it is in %s status"
% (volume_id, volume["status"])
)
LOG.info(reason)
self.result.add_failed_backup(project_id, volume_id, reason)
return res
except OpenstackResourceNotFound:
return False
def replacement_start_response(status, headers, exc_info=None):
"""Overrides the default response to make errors parsable."""
try:
status_code = int(status.split(" ")[0])
state["status_code"] = status_code
except (ValueError, TypeError): # pragma: nocover
raise Exception(
_("ErrorDocumentMiddleware received an invalid "
"status %s") % status)
else:
if (state["status_code"] // 100) not in (2, 3):
# Remove some headers so we can replace them later
# when we have the full error message and can
# compute the length.
headers = [(h, v) for (h, v) in headers
if h not in ("Content-Length", "Content-Type")]
# Save the headers in case we need to modify them.
state["headers"] = headers
return start_response(status, headers, exc_info)
def get_threshold_strtime(self):
time_delta_dict = xtime.parse_timedelta_string(
CONF.conductor.retention_time)
if time_delta_dict is None:
LOG.info(
_("Retention time format is invalid. "
"Follow <YEARS>y<MONTHS>m<WEEKS>w<DAYS>d<HOURS>h<MINUTES>min<SECONDS>s."
))
return None
res = xtime.timeago(
years=time_delta_dict["years"],
months=time_delta_dict["months"],
weeks=time_delta_dict["weeks"],
days=time_delta_dict["days"],
hours=time_delta_dict["hours"],
minutes=time_delta_dict["minutes"],
seconds=time_delta_dict["seconds"],
)
return res.strftime(xtime.DEFAULT_TIME_FORMAT)
def get_id(source_uuid):
"""Derive a short (12 character) id from a random UUID.
The supplied UUID must be a version 4 UUID object.
"""
if isinstance(source_uuid, six.string_types):
source_uuid = uuid.UUID(source_uuid)
if source_uuid.version != 4:
raise ValueError(_("Invalid UUID version (%d)") % source_uuid.version)
# The "time" field of a v4 UUID contains 60 random bits
# (see RFC4122, Section 4.4)
random_bytes = _to_byte_string(source_uuid.time, 60)
# The first 12 bytes (= 60 bits) of base32-encoded output is our data
encoded = base64.b32encode(six.b(random_bytes))[:12]
if six.PY3:
return encoded.lower().decode("utf-8")
else:
return encoded.lower()
def check_instance_volumes(self):
"""Get the list of all the volumes from the project using openstacksdk
Function first list all the servers in the project and get the volumes
that are attached to the instance.
"""
queues_map = []
self.refresh_openstacksdk()
projects = self.openstacksdk.get_projects()
for project in projects:
empty_project = True
self.project_list[project.id] = project
try:
servers = self.openstacksdk.get_servers(project_id=project.id)
except OpenstackHttpException as ex:
LOG.warn(
_(
"Failed to list servers in project %s. %s"
% (project.id, str(ex))
)
)
continue
for server in servers:
if not self.filter_by_server_metadata(server.metadata):
continue
if empty_project:
empty_project = False
self.result.add_project(project.id, project.name)
for volume in server.attached_volumes:
if not self.filter_by_volume_status(volume["id"], project.id):
continue
queues_map.append(
QueueMapping(
project_id=project.id,
volume_id=volume["id"],
backup_id="NULL",
instance_id=server.id,
backup_status=constants.BACKUP_PLANNED,
)
)
return queues_map
from oslo_config import cfg
from staffeln.common import constants
from staffeln.i18n import _
conductor_group = cfg.OptGroup(
"conductor",
title="Conductor Options",
help=_("Options under this group are used " "to define Conductor's configuration."),
)
backup_opts = [
cfg.IntOpt(
"backup_workers",
default=1,
help=_(
"The maximum number of backup processes to "
"fork and run. Default to number of CPUs on the host."
),
),
cfg.IntOpt(
"backup_service_period",
default=30,
min=10,
help=_("The time of bakup period, the unit is one minute."),
),
cfg.StrOpt(
"backup_cycle_timout",
regex=(
r"((?P<years>\d+?)y)?((?P<months>\d+?)mon)?((?P<weeks>\d+?)w)?"
r"((?P<days>\d+?)d)?((?P<hours>\d+?)h)?((?P<minutes>\d+?)min)?((?P<seconds>\d+?)s)?"
),
from oslo_config import cfg
from staffeln.i18n import _
api_group = cfg.OptGroup(
"api",
title="API options",
help=_("Options under this group are used to define staffeln API."),
)
connection_opts = [
cfg.StrOpt(
"host",
default="0.0.0.0",
help=_("IP address on which the staffeln API will listen."),
),
cfg.PortOpt(
"port",
default=8808,
help=_(
"Staffeln API listens on this port number for incoming requests."),
),
cfg.BoolOpt("enabled_ssl", default=False, help=_("ssl enabled")),
cfg.StrOpt("ssl_key_file", default=False, help=_("ssl key file path")),
cfg.StrOpt("ssl_cert_file", default=False, help=_("ssl cert file path")),
]
API_OPTS = connection_opts
def register_opts(conf):
conf.register_group(api_group)
from oslo_config import cfg
from oslo_db import options as oslo_db_options
from staffeln.conf import paths
from staffeln.i18n import _
_DEFAULT_SQL_CONNECTION = "sqlite:///{0}".format(
paths.state_path_def("staffeln.sqlite")
)
database = cfg.OptGroup(
"database",
title="Database options",
help=_("Options under this group are used for defining database."),
)
SQL_OPTS = [
cfg.StrOpt("mysql_engine", default="InnoDB", help=_("MySQL engine to use.")),
]
def register_opts(conf):
oslo_db_options.set_defaults(conf, connection=_DEFAULT_SQL_CONNECTION)
conf.register_group(database)
conf.register_opts(SQL_OPTS, group=database)
def list_opts():
return [(database, SQL_OPTS)]
import os
from oslo_config import cfg
from staffeln.i18n import _
PATH_OPTS = [
cfg.StrOpt(
"pybasedir",
default=os.path.abspath(os.path.join(os.path.dirname(__file__), "../")),
help=_("Directory where the staffeln python module is installed."),
),
cfg.StrOpt(
"bindir",
default="$pybasedir/bin",
help=_("Directory where staffeln binaries are installed."),
),
cfg.StrOpt(
"state_path",
default="$pybasedir",
help=_("Top-level directory for maintaining staffeln's state."),
),
]
def basedir_def(*args):
"""Return an uninterpolated path relative to $pybasedir."""
return os.path.join("$pybasedir", *args)
def bindir_def(*args):
"""Return an uninterpolated path relative to $bindir."""
def _update_task_queue(self):
LOG.info(_("Updating backup task queue..."))
self.controller.refresh_openstacksdk()
self.controller.refresh_backup_result()
current_tasks = self.controller.get_queues()
self.controller.create_queue(current_tasks)
from oslo_config import cfg
from staffeln.i18n import _
notify_group = cfg.OptGroup(
"notification",
title="Notification options",
help=_(
"Options under this group are used to define notification settings."),
)
email_opts = [
cfg.ListOpt(
"receiver",
default=[],
help=_(
"The receivers of the bakcup result by email."
"A list of addresses to receive backup result emails to. A bare"
" string will be treated as a list with 1 address."),
),
cfg.StrOpt(
"sender_email",
help=_("Log in on an SMTP server that requires authentication."
"The user name to authenticate with."),
),
# We can remove the sender password as we are using postfix to send mail and we won't be authenticating.
cfg.StrOpt(
"sender_pwd",
help=_("Log in on an SMTP server that requires authentication."
"The password for the authentication."),
),
cfg.StrOpt(
