def add_stream(self, stream, package_name, headers=None):
"""
:param stream: data
:param package_name: path
:param headers: backup metadata information
:return:
"""
tmpdir = self._create_tempdir()
LOG.info('add stream')
try:
split = package_name.rsplit('/', 1)
# create backup_basedir
backup_basedir = "{0}/{1}".format(self.storage_path,
package_name)
self.create_dirs(backup_basedir)
# define backup_data_name
backup_basepath = "{0}/{1}".format(backup_basedir,
split[0])
backup_metadata = "%s/metadata" % backup_basedir
# write backup to backup_basepath
data_backup = utils.path_join(tmpdir, "data_backup")
with open(data_backup, 'wb') as backup_file:
for el in stream:
backup_file.write(el)
self.put_file(data_backup, backup_basepath)
# write data matadata to backup_metadata
metadata = utils.path_join(tmpdir, "metadata")
with open(metadata, 'wb') as backup_meta:
backup_meta.write(json.dumps(headers))
self.put_file(metadata, backup_metadata)
finally:
shutil.rmtree(tmpdir)
def find_all(self, hostname_backup_name):
backups = []
backup_dir = utils.path_join(self.storage_directory,
hostname_backup_name)
self.create_dirs(backup_dir)
timestamps = self.listdir(backup_dir)
for timestamp in timestamps:
increments = \
self.listdir(utils.path_join(backup_dir, timestamp))
backups.extend(base.Backup.parse_backups(increments, self))
return backups
def find_all(self, hostname_backup_name):
backups = []
backup_dir = utils.path_join(self.storage_directory,
hostname_backup_name)
self.create_dirs(backup_dir)
timestamps = self.listdir(backup_dir)
for timestamp in timestamps:
increments = \
self.listdir(utils.path_join(backup_dir, timestamp))
backups.extend(base.Backup.parse_backups(increments, self))
return backups
def write_backup(self, rich_queue, backup):
"""
Stores backup in storage
:type rich_queue: freezer.streaming.RichQueue
:type backup: freezer.storage.base.Backup
"""
try:
tmpdir = tempfile.mkdtemp()
except Exception:
LOG.error("Unable to create a tmp directory")
raise
try:
data_meta = utils.path_join(tmpdir, "data_meta")
LOG.info("ftp write data_meta %s" % data_meta)
backup = backup.copy(storage=self)
path = backup.data_path
self.create_dirs(path.rsplit('/', 1)[0])
with open(data_meta, mode='wb') as b_file:
for message in rich_queue.get_messages():
b_file.write(message)
self.put_file(data_meta, path)
finally:
shutil.rmtree(tmpdir)
def backup_to_file_path(self, backup):
"""
:param backup:
:type backup: freezer.storage.base.Backup
:return:
"""
return utils.path_join(self._zero_backup_dir(backup), backup)
def backup_to_file_path(self, backup):
"""
:param backup:
:type backup: freezer.storage.base.Backup
:return:
"""
return utils.path_join(self._zero_backup_dir(backup), backup)
def _zero_backup_dir(self, backup):
"""
:param backup:
:type backup: freezer.storage.base.Backup
:return:
"""
return utils.path_join(self.storage_directory,
backup.hostname_backup_name,
backup.full_backup.timestamp)
def rmtree(self, path):
files = self.ftp.listdir(path=path)
for f in files:
filepath = utils.path_join(path, f)
if self._is_dir(filepath):
self.rmtree(filepath)
else:
self.ftp.remove(filepath)
self.ftp.rmdir(path)
def _zero_backup_dir(self, backup):
"""
:param backup:
:type backup: freezer.storage.base.Backup
:return:
"""
return utils.path_join(self.storage_directory,
backup.hostname_backup_name,
backup.full_backup.timestamp)
def rmtree(self, path):
files = self.ftp.listdir(path=path)
for f in files:
filepath = utils.path_join(path, f)
if self._is_dir(filepath):
self.rmtree(filepath)
else:
self.ftp.remove(filepath)
self.ftp.rmdir(path)
def upload_meta_file(self, backup, meta_file):
"""
:param backup:
:type backup: freezer.storage.base.Backup
:param meta_file:
:return:
"""
zero_backup = self._zero_backup_dir(backup)
to_path = utils.path_join(zero_backup, backup.tar())
self.put_file(meta_file, to_path)
def upload_meta_file(self, backup, meta_file):
"""
:param backup:
:type backup: freezer.storage.base.Backup
:param meta_file:
:return:
"""
zero_backup = self._zero_backup_dir(backup)
to_path = utils.path_join(zero_backup, backup.tar())
self.put_file(meta_file, to_path)
def download_meta_file(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:type backup: freezer.storage.base.Backup
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return utils.path_join(self.work_dir, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
if not meta_backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
to_path = utils.path_join(self.work_dir, meta_backup.tar())
if os.path.exists(to_path):
os.remove(to_path)
meta_backup.storage.get_file(
meta_backup.storage.meta_file_abs_path(meta_backup), to_path)
return to_path
def download_meta_file(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:type backup: freezer.storage.base.Backup
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return utils.path_join(self.work_dir, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
if not meta_backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
to_path = utils.path_join(self.work_dir, meta_backup.tar())
if os.path.exists(to_path):
os.remove(to_path)
meta_backup.storage.get_file(
meta_backup.storage.meta_file_abs_path(meta_backup), to_path)
return to_path
def rmtree(self, path):
LOG.info("ftp rmtree path=%s" % path)
files = []
self.ftp.dir(path, files.append)
LOG.info('rm files=%s' % files)
for f in files:
attr = f.split()[0]
file_name = f.split()[-1]
filepath = utils.path_join(path, file_name)
if attr.startswith('d'):
self.rmtree(filepath)
else:
self.ftp.delete(filepath)
self.ftp.rmd(path)
def backup_blocks(self, backup):
LOG.info("ftp backup_blocks ")
self.init()
# should recreate ssh for new process
tmpdir = self._create_tempdir()
try:
data = utils.path_join(tmpdir, "data")
LOG.info("backup_blocksa datadown=%s" % data)
self.get_file(backup.data_path, data)
with open(data, 'rb') as backup_file:
while True:
chunk = backup_file.read(self.max_segment_size)
if chunk == '':
break
if len(chunk):
yield chunk
finally:
shutil.rmtree(tmpdir)
def __init__(self,
engine,
hostname_backup_name,
level_zero_timestamp,
timestamp,
level,
storage=None):
"""
:type storage: freezer.storage.physical.PhysicalStorage
:param hostname_backup_name: name (hostname_backup_name) of backup
:type hostname_backup_name: str
:param timestamp: timestamp of backup (when it was executed)
:type timestamp: int
:param level: current incremental level of backup
:type level: int
:param tar_meta: Is backup has or has not an attached meta
tar file in storage. Default = False
:type tar_meta: bool
:return:
"""
self.hostname_backup_name = hostname_backup_name
self.timestamp = timestamp
self.level = level
self.engine = engine
self.storage = storage
self.level_zero_timestamp = level_zero_timestamp
if storage:
self.increments_data_path = utils.path_join(
self.storage.storage_path, "data", self.engine.name,
self.hostname_backup_name, self.level_zero_timestamp)
self.increments_metadata_path = utils.path_join(
self.storage.storage_path, "metadata", self.engine.name,
self.hostname_backup_name, self.level_zero_timestamp)
self.data_prefix_path = utils.path_join(
self.increments_data_path,
"{0}_{1}".format(self.level, self.timestamp))
self.engine_metadata_path = utils.path_join(
self.data_prefix_path, "engine_metadata")
self.metadata_path = utils.path_join(
self.increments_metadata_path,
"{0}_{1}".format(self.level, self.timestamp), "metadata")
self.data_path = utils.path_join(self.data_prefix_path, "data")
self.segments_path = utils.path_join(self.data_prefix_path,
"segments")
def __init__(self, engine, hostname_backup_name,
level_zero_timestamp, timestamp, level, storage=None):
"""
:type storage: freezer.storage.physical.PhysicalStorage
:param hostname_backup_name: name (hostname_backup_name) of backup
:type hostname_backup_name: str
:param timestamp: timestamp of backup (when it was executed)
:type timestamp: int
:param level: current incremental level of backup
:type level: int
:return:
"""
self.hostname_backup_name = hostname_backup_name
self.timestamp = timestamp
self.level = level
self.engine = engine
self.storage = storage
self.level_zero_timestamp = level_zero_timestamp
if storage:
self.increments_data_path = utils.path_join(
self.storage.storage_path, "data", self.engine.name,
self.hostname_backup_name, self.level_zero_timestamp)
self.increments_metadata_path = utils.path_join(
self.storage.storage_path, "metadata", self.engine.name,
self.hostname_backup_name, self.level_zero_timestamp)
self.data_prefix_path = utils.path_join(
self.increments_data_path,
"{0}_{1}".format(self.level, self.timestamp))
self.engine_metadata_path = utils.path_join(
self.data_prefix_path, "engine_metadata")
self.metadata_path = utils.path_join(
self.increments_metadata_path,
"{0}_{1}".format(self.level, self.timestamp), "metadata")
self.data_path = utils.path_join(self.data_prefix_path, "data")
self.segments_path = utils.path_join(self.data_prefix_path,
"segments")
def meta_file_abs_path(self, backup):
zero_backup = self._zero_backup_dir(backup)
return utils.path_join(zero_backup, backup.tar())
def _create_image(self, path, restore_from_timestamp):
"""
:param path:
:param restore_from_timestamp:
:type restore_from_timestamp: int
:return:
"""
backup = self._get_backups(path, restore_from_timestamp)
if self.storage.type == 'swift':
swift = self.client_manager.get_swift()
path = "{0}_segments/{1}/{2}".format(self.container, path, backup)
stream = swift.get_object(self.container,
"{}/{}".format(path, backup),
resp_chunk_size=10000000)
length = int(stream[0]["x-object-meta-length"])
data = utils.ReSizeStream(stream[1], length, 1)
info = stream[0]
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
elif self.storage.type == 's3':
if self.storage.get_object_prefix() != '':
base_path = "{0}/{1}/{2}".format(
self.storage.get_object_prefix(), path, backup)
else:
base_path = "{0}/{1}".format(path, backup)
image_file = "{0}/{1}".format(base_path, path)
s3_object = self.storage.get_object(
bucket_name=self.storage.get_bucket_name(), key=image_file)
stream = utils.S3ResponseStream(data=s3_object['Body'],
chunk_size=10000000)
data = utils.ReSizeStream(stream, s3_object['ContentLength'], 1)
metadata = "{0}/metadata".format(base_path)
metadata_object = self.storage.get_object(
bucket_name=self.storage.get_bucket_name(), key=metadata)
info = json.load(metadata_object['Body'])
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
elif self.storage.type == 'local':
image_file = "{0}/{1}/{2}/{3}".format(self.container, path, backup,
path)
metadata_file = "{0}/{1}/{2}/metadata".format(
self.container, path, backup)
try:
data = open(image_file, 'rb')
except Exception:
msg = "Failed to open image file {}".format(image_file)
LOG.error(msg)
raise BaseException(msg)
info = json.load(open(metadata_file, 'r'))
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
elif self.storage.type == 'ssh':
image_file = "{0}/{1}/{2}/{3}".format(self.container, path, backup,
path)
metadata_file = "{0}/{1}/{2}/metadata".format(
self.container, path, backup)
try:
data = self.storage.open(image_file, 'rb')
except Exception:
msg = "Failed to open remote image file {}".format(image_file)
LOG.error(msg)
raise BaseException(msg)
info = json.loads(self.storage.read_metadata_file(metadata_file))
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
elif self.storage.type in ['ftp', 'ftps']:
image_file = "{0}/{1}/{2}/{3}".format(self.container, path, backup,
path)
metadata_file = "{0}/{1}/{2}/metadata".format(
self.container, path, backup)
try:
tmpdir = tempfile.mkdtemp()
except Exception:
LOG.error("Unable to create a tmp directory")
raise
try:
data_image = utils.path_join(tmpdir, "data_image")
LOG.info('create image restore ftp storage')
self.storage.get_file(image_file, data_image)
data_meta = utils.path_join(tmpdir, "data_meta")
self.storage.get_file(metadata_file, data_meta)
data = open(data_image, 'rb')
info = json.load(open(data_meta, 'r'))
image = self.client_manager.create_image(
name="restore_{}".format(path),
container_format="bare",
disk_format="raw",
data=data)
return info, image
finally:
shutil.rmtree(tmpdir)
else:
return {}
def meta_file_abs_path(self, backup):
zero_backup = self._zero_backup_dir(backup)
return utils.path_join(zero_backup, backup.tar())
def metadata_path(self, engine, hostname_backup_name):
return utils.path_join(self.storage_path, "metadata", engine.name,
hostname_backup_name)
def metadata_path(self, engine, hostname_backup_name):
return utils.path_join(self.storage_path, "metadata", engine.name,
hostname_backup_name)
def backup(self, backup_resource, hostname_backup_name, no_incremental,
max_level, always_level, restart_always_level, queue_size=2):
"""
Here we now location of all interesting artifacts like metadata
Should return stream for storing data.
:return: stream
"""
prev_backup = self.storage.previous_backup(
engine=self,
hostname_backup_name=hostname_backup_name,
no_incremental=no_incremental,
max_level=max_level,
always_level=always_level,
restart_always_level=restart_always_level
)
try:
tmpdir = tempfile.mkdtemp()
except Exception:
LOG.error("Unable to create a tmp directory")
raise
try:
engine_meta = utils.path_join(tmpdir, "engine_meta")
freezer_meta = utils.path_join(tmpdir, "freezer_meta")
if prev_backup:
prev_backup.storage.get_file(prev_backup.engine_metadata_path,
engine_meta)
timestamp = utils.DateTime.now().timestamp
level_zero_timestamp = (prev_backup.level_zero_timestamp
if prev_backup else timestamp)
backup = base.Backup(
engine=self,
hostname_backup_name=hostname_backup_name,
level_zero_timestamp=level_zero_timestamp,
timestamp=timestamp,
level=(prev_backup.level + 1 if prev_backup else 0)
)
input_queue = streaming.RichQueue(queue_size)
read_except_queue = queue.Queue()
write_except_queue = queue.Queue()
read_stream = streaming.QueuedThread(
self.backup_stream,
input_queue,
read_except_queue,
kwargs={"backup_resource": backup_resource,
"manifest_path": engine_meta})
write_stream = streaming.QueuedThread(
self.storage.write_backup,
input_queue,
write_except_queue,
kwargs={"backup": backup})
read_stream.daemon = True
write_stream.daemon = True
read_stream.start()
write_stream.start()
read_stream.join()
write_stream.join()
# queue handling is different from SimpleQueue handling.
def handle_except_queue(except_queue):
if not except_queue.empty():
while not except_queue.empty():
e = except_queue.get_nowait()
LOG.critical('Engine error: {0}'.format(e))
return True
else:
return False
got_exception = None
got_exception = (handle_except_queue(read_except_queue) or
got_exception)
got_exception = (handle_except_queue(write_except_queue) or
got_exception)
if got_exception:
raise engine_exceptions.EngineException(
"Engine error. Failed to backup.")
with open(freezer_meta, mode='wb') as b_file:
b_file.write(json.dumps(self.metadata(backup_resource)))
self.storage.put_metadata(engine_meta, freezer_meta, backup)
finally:
shutil.rmtree(tmpdir)
