def start_backup_job(self):
jobmessage(
M_INFO,
"Start backup, try to connect to %s:%s"
% (self.options["host"], self.options["port"]),
)
if BareosLibcloudApi.probe_driver(self.options) == "failed":
jobmessage(
M_FATAL,
"Could not connect to libcloud driver: %s:%s"
% (self.options["host"], self.options["port"]),
)
return bRC_Error
jobmessage(
M_INFO, "Connected, last backup: %s (ts: %s)" % (self.last_run, self.since),
)
try:
self.api = BareosLibcloudApi(
self.options,
self.last_run,
self.options["temporary_download_directory"],
)
debugmessage(100, "BareosLibcloudApi started")
except Exception as e:
debugmessage(100, "Error: %s" % e)
jobmessage(M_FATAL, "Starting BareosLibcloudApi failed: %s" % e)
return bRC_Cancel
return bRC_OK
def check_worker_messages(self):
while not self.message_queue.empty():
try:
message = self.message_queue.get_nowait()
message_text = ("%3d: %s") % (
message.worker_id,
message.message_string,
)
if message.type == MESSAGE_TYPE.INFO_MESSAGE:
jobmessage(M_INFO, message_text)
elif message.type == MESSAGE_TYPE.ERROR_MESSAGE:
jobmessage(M_ERROR, message_text)
elif message.type == MESSAGE_TYPE.READY_MESSAGE:
if message.worker_id == 0:
self.count_bucket_explorer_ready += 1
else:
self.count_worker_ready += 1
elif message.type == MESSAGE_TYPE.ABORT_MESSAGE:
return ABORT
elif message.type == MESSAGE_TYPE.DEBUG_MESSAGE:
debugmessage(message.level, message_text)
else:
raise Exception(value="Unknown message type")
except Exception as e:
jobmessage(M_INFO,
"check_worker_messages exception: %s" % str(e))
return SUCCESS
def parse_plugin_definition(self, plugindef):
debugmessage(100, "parse_plugin_definition()")
config_filename = self.options.get("config_file")
if config_filename:
if self.__parse_config_file(config_filename):
return bRC_OK
debugmessage(100, "Could not load configfile %s" % (config_filename))
jobmessage(M_FATAL, "Could not load configfile %s" % (config_filename))
return bRC_Error
def create_file(self, restorepkt):
debugmessage(
100, "create_file() entry point in Python called with %s\n" % (restorepkt)
)
FNAME = FilenameConverter.BackupToBucket(restorepkt.ofname)
dirname = StringCodec.encode_for_restore(os.path.dirname(FNAME))
if not os.path.exists(dirname):
jobmessage(M_INFO, "Directory %s does not exist, creating it\n" % dirname)
os.makedirs(dirname)
if restorepkt.type == FT_REG:
restorepkt.create_status = CF_EXTRACT
return bRC_OK
def _remove_tmp_dir(self):
debugmessage(
100, "Try to remove leftover files from: %s" % (self.tmp_dir_path))
try:
files = glob.glob(self.tmp_dir_path + "/*")
cnt = 0
for f in files:
os.remove(f)
cnt += 1
if cnt != 0:
debugmessage(
100,
"Removed %d leftover files from: %s" %
(cnt, self.tmp_dir_path),
)
else:
debugmessage(
100,
"No leftover files to remove from: %s" %
(self.tmp_dir_path),
)
os.rmdir(self.tmp_dir_path)
debugmessage(
100, "Removed temporary directory: %s" % (self.tmp_dir_path))
except:
pass
def parse_plugin_definition(self, plugindef):
if version_info.major >= 3 and version_info.minor > 7:
jobmessage(
M_FATAL,
"Need Python version < 3.8 for Bareos Libcloud (current version: %d.%d.%d)"
% (
version_info.major,
version_info.minor,
version_info.micro,
),
)
return bRC_Error
debugmessage(100, "parse_plugin_definition()")
config_filename = self.options.get("config_file")
if config_filename:
if self._parse_config_file(config_filename):
return bRC_OK
debugmessage(100, "Could not load configfile %s" % (config_filename))
jobmessage(M_FATAL, "Could not load configfile %s" % (config_filename))
return bRC_Error
def __init__(self, plugindef):
debugmessage(
100, "BareosFdPluginLibcloud called with plugindef: %s" % (plugindef,)
)
super(BareosFdPluginLibcloud, self).__init__(plugindef)
super(BareosFdPluginLibcloud, self).parse_plugin_definition(plugindef)
self.options["treat_download_errors_as_warnings"] = False
self.__parse_options()
self.last_run = datetime.datetime.fromtimestamp(self.since)
self.last_run = self.last_run.replace(tzinfo=None)
# The backup task in process
# Set to None when the whole backup is completed
# Restore's path will not touch this
self.current_backup_task = {}
self.number_of_objects_to_backup = 0
self.FILE = None
self.active = True
self.api = None
def __parse_config_file(self, config_filename):
"""
Parse the config file given in the config_file plugin option
"""
debugmessage(100, "parse_config_file(): parse %s\n" % (config_filename))
self.config = configparser.ConfigParser()
try:
if version_info[:3] < (3, 2):
self.config.readfp(open(config_filename))
else:
self.config.read_file(open(config_filename))
except (IOError, OSError) as err:
debugmessage(
100,
"Error reading config file %s: %s\n"
% (self.options["config_file"], err.strerror),
)
return False
return self.__check_config(config_filename)
def shutdown(self):
debugmessage(100, "Shut down worker")
self.bucket_explorer.shutdown()
for w in self.worker:
w.shutdown()
debugmessage(100, "Wait for worker")
while not self.bucket_explorer_ready():
self.check_worker_messages()
while not self.worker_ready():
self.check_worker_messages()
debugmessage(100, "Worker finished")
while not self.discovered_objects_queue.empty():
self.discovered_objects_queue.get()
while not self.downloaded_objects_queue.empty():
self.downloaded_objects_queue.get()
while not self.message_queue.empty():
self.message_queue.get()
debugmessage(100, "Join worker processes")
for w in self.worker:
w.join(timeout=0.3)
self.bucket_explorer.join(timeout=0.3)
for w in self.worker:
if w.is_alive():
w.terminate()
if self.bucket_explorer.is_alive():
self.bucket_explorer.terminate()
try:
self.__remove_tmp_dir()
except:
pass
jobmessage(M_INFO, "Finished shutdown of worker processes")
def __shutdown(self):
self.active = False
jobmessage(
M_INFO,
"BareosFdPluginLibcloud finished with %d files"
% (self.number_of_objects_to_backup),
)
if self.api == None:
debugmessage(100, "BareosLibcloudApi did not initialize properly")
else:
debugmessage(100, "Shut down BareosLibcloudApi")
self.api.shutdown()
debugmessage(100, "BareosLibcloudApi is shut down")
def plugin_io(self, IOP):
if self.current_backup_task is None:
return bRC_Error
if IOP.func == IO_OPEN:
# Only used by the 'restore' path
if IOP.flags & (os.O_CREAT | os.O_WRONLY):
self.FILE = open(
StringCodec.encode_for_restore(
FilenameConverter.BackupToBucket(IOP.fname)
),
"wb",
)
return bRC_OK
elif IOP.func == IO_READ:
IOP.buf = bytearray(IOP.count)
IOP.io_errno = 0
if self.FILE is None:
return bRC_Error
try:
buf = self.FILE.read(IOP.count)
IOP.buf[:] = buf
IOP.status = len(buf)
return bRC_OK
except IOError as e:
jobmessage(
M_ERROR,
"Cannot read from %s/%s: %s"
% (
self.current_backup_task["bucket"],
self.current_backup_task["name"],
e,
),
)
IOP.status = 0
if self.options["treat_download_errors_as_warnings"]:
return bRC_Skip
else:
return bRC_Error
elif IOP.func == IO_WRITE:
try:
self.FILE.write(IOP.buf)
IOP.status = IOP.count
IOP.io_errno = 0
except IOError as msg:
IOP.io_errno = -1
jobmessage(M_ERROR, "Failed to write data: %s" % (msg,))
return bRC_OK
elif IOP.func == IO_CLOSE:
if self.FILE:
self.FILE.close()
if "type" in self.current_backup_task:
if self.current_backup_task["type"] == TASK_TYPE.TEMP_FILE:
debugmessage(
110,
"Removing temporary file: %s"
% (self.current_backup_task["tmpfile"]),
)
try:
os.remove(self.current_backup_task["tmpfile"])
except:
debugmessage(
110,
"Could not remove temporary file: %s"
% (self.current_backup_task["tmpfile"]),
)
return bRC_OK
return bRC_OK
def start_backup_file(self, savepkt):
error = False
while self.active:
worker_result = self.api.check_worker_messages()
if worker_result == ERROR:
if self.options["treat_download_errors_as_warnings"]:
pass
else:
self.active = False
error = True
elif worker_result == ABORT:
self.active = False
error = True
else:
self.current_backup_task = self.api.get_next_task()
if self.current_backup_task != None:
break
elif self.api.worker_ready():
self.active = False
else:
sleep(0.01)
if not self.active:
self.__shutdown()
savepkt.fname = "" # dummy value
if error:
jobmessage(M_FATAL, "Shutdown after worker error")
return bRC_Cancel
else:
return bRC_Skip
filename = FilenameConverter.BucketToBackup(
"%s/%s"
% (self.current_backup_task["bucket"], self.current_backup_task["name"],)
)
debugmessage(100, "Backup file: %s" % (filename,))
statp = bareosfd.StatPacket()
# statp.size = self.current_backup_task["size"]
# statp.mtime = self.current_backup_task["mtime"]
# statp.atime = 0
# statp.ctime = 0
savepkt.statp = statp
savepkt.fname = StringCodec.encode_for_backup(filename)
savepkt.type = FT_REG
if self.current_backup_task["type"] == TASK_TYPE.DOWNLOADED:
self.FILE = self.current_backup_task["data"]
elif self.current_backup_task["type"] == TASK_TYPE.TEMP_FILE:
try:
self.FILE = io.open(self.current_backup_task["tmpfile"], "rb")
except Exception as e:
jobmessage(M_FATAL, "Could not open temporary file for reading.")
self.__shutdown()
return bRC_Error
elif self.current_backup_task["type"] == TASK_TYPE.STREAM:
try:
self.FILE = IterStringIO(self.current_backup_task["data"].as_stream())
except ObjectDoesNotExistError:
if self.options["treat_download_errors_as_warnings"]:
jobmessage(
M_WARNING,
"Skipped file %s because it does not exist anymore"
% (self.current_backup_task["name"]),
)
return bRC_Skip
else:
jobmessage(
M_ERROR,
"File %s does not exist anymore"
% (self.current_backup_task["name"]),
)
return bRC_Error
else:
raise Exception(value='Wrong argument for current_backup_task["type"]')
return bRC_OK
def __check_config(self, config_filename):
"""
Check the configuration and set or override options if necessary,
considering mandatory: username and password in the [credentials] section
"""
mandatory_sections = ["credentials", "host", "misc"]
mandatory_options = {}
mandatory_options["credentials"] = ["username", "password"]
mandatory_options["host"] = ["hostname", "port", "provider", "tls"]
mandatory_options["misc"] = [
"nb_worker",
"queue_size",
"prefetch_size",
"temporary_download_directory",
]
optional_options = {}
optional_options["misc"] = ["treat_download_errors_as_warnings"]
# this maps config file options to libcloud options
option_map = {
"hostname": "host",
"port": "port",
"provider": "provider",
"username": "******",
"password": "******",
}
for section in mandatory_sections:
if not self.config.has_section(section):
debugmessage(
100,
"BareosFdPluginLibcloud: Section [%s] missing in config file %s\n"
% (section, config_filename),
)
return False
for option in mandatory_options[section]:
if not self.config.has_option(section, option):
debugmessage(
100,
"BareosFdPluginLibcloud: Option [%s] missing in Section %s\n"
% (option, section),
)
return False
value = self.config.get(section, option)
try:
if option == "tls":
self.options["secure"] = strtobool(value)
elif option == "nb_worker":
self.options["nb_worker"] = int(value)
elif option == "queue_size":
self.options["queue_size"] = int(value)
elif option == "prefetch_size":
self.options["prefetch_size"] = eval(value)
elif option == "temporary_download_directory":
self.options["temporary_download_directory"] = value
else:
self.options[option_map[option]] = value
except:
debugmessage(
100,
"Could not evaluate: %s in config file %s"
% (value, config_filename),
)
return False
for option in optional_options["misc"]:
if self.config.has_option(section, option):
try:
value = self.config.get(section, option)
self.options["treat_download_errors_as_warnings"] = strtobool(value)
except:
debugmessage(
100,
"Could not evaluate: %s in config file %s"
% (value, config_filename),
)
return False
return True
def plugin_io(self, IOP):
if self.current_backup_task is None:
return bRC_Error
if IOP.func == IO_OPEN:
# Only used by the 'restore' path
if IOP.flags & (os.O_CREAT | os.O_WRONLY):
self.FILE = open(
StringCodec.encode_for_restore(
FilenameConverter.BackupToBucket(IOP.fname)
),
"wb",
)
return bRC_OK
elif IOP.func == IO_READ:
IOP.buf = bytearray(IOP.count)
IOP.io_errno = 0
if self.FILE is None:
return bRC_Error
try:
if self.current_backup_task["type"] == TASK_TYPE.STREAM:
if self.current_backup_task["skip_file"]:
return bRC_Skip # workaround for self.start_backup_file
buf = self.FILE.read(IOP.count)
IOP.buf[:] = buf
IOP.status = len(buf)
if self.current_backup_task["type"] == TASK_TYPE.STREAM:
self.current_backup_task["stream_length"] += len(buf)
return bRC_OK
except IOError as e:
jobmessage(
M_ERROR,
"Cannot read from %s/%s: %s"
% (
self.current_backup_task["bucket"],
self.current_backup_task["name"],
e,
),
)
IOP.status = 0
if self.options["fail_on_download_error"]:
return bRC_Error
else:
return bRC_Skip
elif IOP.func == IO_WRITE:
try:
self.FILE.write(IOP.buf)
IOP.status = IOP.count
IOP.io_errno = 0
except IOError as msg:
IOP.io_errno = -1
jobmessage(M_ERROR, "Failed to write data: %s" % (msg,))
return bRC_OK
elif IOP.func == IO_CLOSE:
if self.FILE:
self.FILE.close()
if "type" in self.current_backup_task:
if self.current_backup_task["type"] == TASK_TYPE.TEMP_FILE:
debugmessage(
110,
"Removing temporary file: %s"
% (self.current_backup_task["tmpfile"]),
)
try:
os.remove(self.current_backup_task["tmpfile"])
except:
debugmessage(
110,
"Could not remove temporary file: %s"
% (self.current_backup_task["tmpfile"]),
)
elif self.current_backup_task["type"] == TASK_TYPE.STREAM:
if (
self.current_backup_task["stream_length"]
!= self.current_backup_task["size"]
):
level = M_ERROR
ret = bRC_Skip
if self.options["fail_on_download_error"]:
level = M_FATAL
ret = bRC_Error
jobmessage(
level,
"skipping file %s that has %d bytes, "
"not %d bytes as stated before"
% (
self.current_backup_task["name"],
self.current_backup_task["stream_length"],
self.current_backup_task["size"],
),
)
return ret
return bRC_OK
return bRC_OK
def _create_tmp_dir(self):
debugmessage(
100, "Try to create temporary directory: %s" % (self.tmp_dir_path))
os.makedirs(self.tmp_dir_path)
debugmessage(100,
"Created temporary directory: %s" % (self.tmp_dir_path))
