def existing_backups_for_fileset(self, fileset, is_full):
"""Retrieve list of existing backups for a single fileset."""
existing_blobs_dict = dict()
marker = None
while True:
results = self.backup_configuration.storage_client.list_blobs(
container_name=self.backup_configuration.
azure_storage_container_name,
prefix=Naming.construct_blobname_prefix(
fileset=fileset,
is_full=is_full,
vmname=self.backup_configuration.get_vm_name()),
marker=marker)
for blob in results:
blob_name = blob.name
parts = Naming.parse_blobname(blob_name)
if parts is None:
continue
start_timestamp = parts[2]
if not existing_blobs_dict.has_key(start_timestamp):
existing_blobs_dict[start_timestamp] = []
existing_blobs_dict[start_timestamp].append(blob_name)
if results.next_marker:
marker = results.next_marker
else:
break
return existing_blobs_dict
def existing_backups(self, filesets=None, container=None):
"""Retrieve list of existing backups. Returns tuples (name, datetime, length)"""
existing_blobs_list = list()
marker = None
results = list(
self.backup_configuration.storage_client.list_blobs(
container_name=container
or self.backup_configuration.azure_storage_container_name,
marker=marker))
results = sorted(results, key=lambda x: x.name)
for blob in results:
blob_name = blob.name
parts = Naming.parse_blobname(blob_name)
if parts is None:
continue
(fileset_of_existing_blob, _is_full, _start_timestamp,
_vmname) = parts
if not filesets or fileset_of_existing_blob in filesets:
existing_blobs_list.append(
(blob_name, blob.properties.creation_time,
blob.properties.content_length))
return existing_blobs_list
def test_restore_single_fileset(self):
"""Test restoring a single fileset."""
# We should have a backup from the preceding test cases.
backups = self.agent.existing_backups_for_fileset('tmp_dir', True)
blob_name = backups.popitem()[1][0]
(fileset, _is_full, timestamp) = Naming.parse_blobname(blob_name)
self.agent.restore_single_fileset(fileset, timestamp, '/tmp')
# TODO: test that expected files were indeed restored...
return True
def test_restore_default_fileset_override_container(self):
"""Test restoring a single fileset and override the container name."""
# We should have a backup from the preceding test cases.
backups = self.agent.existing_backups_for_fileset('fs', True)
blob_name = backups.popitem()[1][0]
(fileset, _is_full, timestamp, vmname) = Naming.parse_blobname(blob_name)
container = self.cfg.azure_storage_container_name
self.agent.restore(timestamp, '/tmp', [], container=container)
# TODO: test that expected files were indeed restored...
return True
def restore_single_fileset(self,
fileset,
restore_point,
output_dir,
stream=False,
container=None):
""" Restore backup for a single fileset."""
vmname = self.backup_configuration.get_vm_name()
blob_to_restore = Naming.construct_blobname(fileset, True,
restore_point, vmname)
self.restore_blob(blob_to_restore, output_dir, stream, container)
def prune_old_backups(self, older_than, filesets):
"""
Delete (prune) old backups from Azure storage.
"""
minimum_deletable_age = datetime.timedelta(7, 0)
logging.warn("Deleting files older than %s", older_than)
if older_than < minimum_deletable_age:
msg = "Will not delete files younger than {}, ignoring".format(
minimum_deletable_age)
logging.warn(msg)
return
marker = None
while True:
results = self.backup_configuration.storage_client.list_blobs(
container_name=self.backup_configuration.
azure_storage_container_name,
marker=marker)
for blob in results:
parts = Naming.parse_blobname(blob.name)
if parts is None:
continue
(fileset, _is_full, start_timestamp, _vmname) = parts
if (fileset != None) and not fileset in filesets:
continue
diff = Timing.time_diff(start_timestamp,
Timing.now_localtime())
delete = diff > older_than
if delete:
logging.warn("Deleting %s", blob.name)
self.backup_configuration.storage_client.delete_blob(
container_name=self.backup_configuration.
azure_storage_container_name,
blob_name=blob.name)
else:
logging.warn("Keeping %s", blob.name)
if results.next_marker:
marker = results.next_marker
else:
break
def backup_single_fileset(self,
fileset,
is_full,
force,
command=None,
rate=None):
"""
Backup a single fileset using the specified command.
If no command is provided, it will be looked up in the config file.
"""
logging.info("Backup request for fileset: %s", fileset)
# Determine if backup can run according to schedule
start_timestamp = Timing.now_localtime()
end_timestamp = None
if not self.should_run_backup(fileset=fileset,
is_full=is_full,
force=force,
start_timestamp=start_timestamp):
logging.warn("Skipping backup of fileset %s", fileset)
return
# Final destination container
dest_container_name = self.backup_configuration.azure_storage_container_name
vmname = self.backup_configuration.get_vm_name()
# Name of the backup blob
blob_name = Naming.construct_blobname(fileset=fileset,
is_full=is_full,
start_timestamp=start_timestamp,
vmname=vmname)
# Command to run to execute the backup
if not command:
command = self.backup_configuration.get_backup_command(fileset)
try:
# Run the backup command
proc = self.executable_connector.run_backup_command(command, rate)
logging.info("Streaming backup to blob: %s in container: %s",
blob_name, dest_container_name)
# Stream backup command stdout to the blob
storage_client = self.backup_configuration.storage_client
storage_client.create_blob_from_stream(
container_name=dest_container_name,
blob_name=blob_name,
stream=proc.stdout,
use_byte_buffer=True,
max_connections=1)
# Wait for the command to terminate
retcode = proc.wait()
# Check return code
# Ignore return code 1 (files changed during backup)
if retcode == 1:
logging.warning("ignoring tar command return code 1")
elif retcode != 0:
raise BackupException(
"tar command failed with return code {}".format(retcode))
except Exception as ex:
logging.error("Failed to stream blob: %s", ex.message)
end_timestamp = Timing.now_localtime()
self.send_notification(is_full=is_full,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
success=False,
blob_size=0,
blob_path='/' + dest_container_name + '/' +
blob_name,
error_msg=ex.message)
raise ex
logging.info("Finished streaming blob: %s", blob_name)
end_timestamp = Timing.now_localtime()
# Get blob size
try:
blob_props = storage_client.get_blob_properties(
dest_container_name, blob_name)
except Exception as ex:
logging.error("Failed to get blob size: %s", ex.message)
self.send_notification(is_full=is_full,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
success=False,
blob_size=0,
blob_path='/' + dest_container_name + '/' +
blob_name,
error_msg=ex.message)
raise ex
# Send notification
self.send_notification(is_full=is_full,
start_timestamp=start_timestamp,
end_timestamp=end_timestamp,
success=True,
blob_size=blob_props.properties.content_length,
blob_path='/' + dest_container_name + '/' +
blob_name,
error_msg=None)
# Return name of new blob
return blob_name