def setUp(self):
opts = osclients.OpenstackOpts.create_from_env().get_opts_dicts()
self.storage = swift.SwiftStorage(
osclients.OSClientManager(opts.pop('auth_url'),
opts.pop('auth_method', 'password'),
**opts
),
"freezer_ops-aw1ops1-gerrit0001.aw1.hpcloud.net",
"/tmp/",
100, skip_prepare=True
)
self.files = [
"tar_metadata_hostname_backup_1000_0",
"hostname_backup_1000_0",
]
self.increments = [
"tar_metadata_hostname_backup_1000_0",
"hostname_backup_1000_0",
"tar_metadata_hostname_backup_2000_1",
"hostname_backup_2000_1",
]
self.cycles_increments = [
"tar_metadata_hostname_backup_1000_0",
"hostname_backup_1000_0",
"tar_metadata_hostname_backup_2000_1",
"hostname_backup_2000_1",
"tar_metadata_hostname_backup_3000_0",
"hostname_backup_3000_0",
"tar_metadata_hostname_backup_4000_1",
"hostname_backup_4000_1",
]
self.backup = base.Backup(self.storage,
"hostname_backup", 1000, tar_meta=True,)
self.backup_2 = base.Backup(self.storage,
"hostname_backup", 3000, tar_meta=True)
self.increment = base.Backup(self.storage,
"hostname_backup", 2000,
full_backup=self.backup,
level=1,
tar_meta=True)
self.increment_2 = base.Backup(self.storage,
"hostname_backup", 4000,
full_backup=self.backup_2,
level=1,
tar_meta=True)
def get_level_zero(self,
engine,
hostname_backup_name,
recent_to_date=None):
"""
Gets backups by backup_name and hostname
:type engine: freezer.engine.engine.BackupEngine
:param engine: Search for backups made by specified engine
:type hostname_backup_name: str
:param hostname_backup_name: Search for backup with specified name
:type recent_to_date: int
:param recent_to_date:
:rtype: list[freezer.storage.base.Backup]
:return: dictionary of level zero timestamps with attached storage
"""
path = self.metadata_path(engine=engine,
hostname_backup_name=hostname_backup_name)
zeros = [
base.Backup(storage=self,
engine=engine,
hostname_backup_name=hostname_backup_name,
level_zero_timestamp=int(t),
timestamp=int(t),
level=0) for t in self.listdir(path)
]
if recent_to_date:
zeros = [
zero for zero in zeros if zero.timestamp >= recent_to_date
]
return zeros
def backup(self, backup_resource, hostname_backup_name, no_incremental,
max_level, always_level, restart_always_level, queue_size=2):
"""
Here we now location of all interesting artifacts like metadata
Should return stream for storing data.
:return: stream
"""
prev_backup = self.storage.previous_backup(
engine=self,
hostname_backup_name=hostname_backup_name,
no_incremental=no_incremental,
max_level=max_level,
always_level=always_level,
restart_always_level=restart_always_level
)
try:
tmpdir = tempfile.mkdtemp()
except Exception:
LOG.error("Unable to create a tmp directory")
raise
try:
engine_meta = utils.path_join(tmpdir, "engine_meta")
freezer_meta = utils.path_join(tmpdir, "freezer_meta")
if prev_backup:
prev_backup.storage.get_file(prev_backup.engine_metadata_path,
engine_meta)
timestamp = utils.DateTime.now().timestamp
level_zero_timestamp = (prev_backup.level_zero_timestamp
if prev_backup else timestamp)
backup = base.Backup(
engine=self,
hostname_backup_name=hostname_backup_name,
level_zero_timestamp=level_zero_timestamp,
timestamp=timestamp,
level=(prev_backup.level + 1 if prev_backup else 0)
)
input_queue = streaming.RichQueue(queue_size)
read_except_queue = queue.Queue()
write_except_queue = queue.Queue()
read_stream = streaming.QueuedThread(
self.backup_stream,
input_queue,
read_except_queue,
kwargs={"backup_resource": backup_resource,
"manifest_path": engine_meta})
write_stream = streaming.QueuedThread(
self.storage.write_backup,
input_queue,
write_except_queue,
kwargs={"backup": backup})
read_stream.daemon = True
write_stream.daemon = True
read_stream.start()
write_stream.start()
read_stream.join()
write_stream.join()
# queue handling is different from SimpleQueue handling.
def handle_except_queue(except_queue):
if not except_queue.empty():
while not except_queue.empty():
e = except_queue.get_nowait()
LOG.critical('Engine error: {0}'.format(e))
return True
else:
return False
got_exception = None
got_exception = (handle_except_queue(read_except_queue) or
got_exception)
got_exception = (handle_except_queue(write_except_queue) or
got_exception)
if got_exception:
raise engine_exceptions.EngineException(
"Engine error. Failed to backup.")
with open(freezer_meta, mode='wb') as b_file:
b_file.write(json.dumps(self.metadata(backup_resource)))
self.storage.put_metadata(engine_meta, freezer_meta, backup)
finally:
shutil.rmtree(tmpdir)