def _download_tar_meta(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:type backup: SwiftBackup
:return:
"""
if not backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
utils.create_dir(self.work_dir)
tar_meta = backup.tar()
tar_meta_abs = "{0}/{1}".format(self.work_dir, tar_meta)
logging.info('[*] Downloading object {0} {1}'.format(
tar_meta, tar_meta_abs))
if os.path.exists(tar_meta_abs):
os.remove(tar_meta_abs)
with open(tar_meta_abs, 'ab') as obj_fd:
iterator = self.swift().get_object(
self.container, tar_meta, resp_chunk_size=16000000)[1]
for obj_chunk in iterator:
obj_fd.write(obj_chunk)
def download_meta_file(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:type backup: freezer.storage.Backup
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return "{0}{1}{2}".format(self.work_dir, os.sep, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
if not meta_backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
tar_meta = meta_backup.tar()
tar_meta_abs = "{0}{1}{2}".format(self.work_dir, os.sep, tar_meta)
logging.info('[*] Downloading object {0} {1}'.format(
tar_meta, tar_meta_abs))
if os.path.exists(tar_meta_abs):
os.remove(tar_meta_abs)
with open(tar_meta_abs, 'ab') as obj_fd:
iterator = self.swift().get_object(
self.container, tar_meta, resp_chunk_size=self.chunk_size)[1]
for obj_chunk in iterator:
obj_fd.write(obj_chunk)
return tar_meta_abs
def lvm_eval(backup_opt_dict):
"""
Evaluate if the backup must be executed using lvm snapshot
or just directly on the plain filesystem. If no lvm options are specified
the backup will be executed directly on the file system and without
use lvm snapshot. If one of the lvm options are set, then the lvm snap
will be used to execute backup. This mean all the required options
must be set accordingly
"""
if not backup_opt_dict.lvm_volgroup:
logging.warning('[*] Required lvm_volgroup not set. The backup will '
'execute without lvm snapshot.')
return False
if not backup_opt_dict.lvm_srcvol:
logging.warning('[*] Required lvm_srcvol not set. The backup will '
'execute without lvm snapshot.')
return False
if not backup_opt_dict.lvm_dirmount:
logging.warning('[*] Required lvm_dirmount not set. The backup will '
'execute without lvm snapshot.')
return False
# Create lvm_dirmount dir if it doesn't exists and write action in logs
create_dir(backup_opt_dict.lvm_dirmount)
return True
def prepare_logging(log_file='~/.freezer/freezer.log'):
"""
Creates log directory and log file if no log files provided
:return:
"""
expanded_file_name = os.path.expanduser(log_file)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
return expanded_file_name
def do_job_download(client, args):
create_dir(args.jobs_dir, do_log=True)
for doc in _job_list(client, args):
fname = os.path.normpath('{0}/job_{1}.conf'.
format(args.jobs_dir, doc['job_id']))
try:
utils.save_doc_to_json_file(doc, fname)
except:
print("Unable to write to file {0}".format(fname))
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
format(dry_run_message)))
return expanded_file_name
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.format(
dry_run_message)))
return expanded_file_name
def create_dirs(self, tmpdir):
tmpdir = tmpdir.strpath
if self.temp:
backup_dir = tempfile.mkdtemp(dir=tmpdir, prefix=self.BACKUP_DIR_PREFIX)
files_dir = tempfile.mkdtemp(dir=tmpdir, prefix=self.FILES_DIR_PREFIX)
else:
backup_dir = tmpdir + self.BACKUP_DIR_PREFIX
files_dir = tmpdir + self.FILES_DIR_PREFIX
utils.create_dir(backup_dir)
utils.create_dir(files_dir)
self.create_content(files_dir)
return backup_dir, files_dir
def test_create_dir(self):
dir1 = '/tmp'
dir2 = '/tmp/testnoexistent1234'
dir3 = '~'
fakeos = Os()
assert utils.create_dir(dir1) is None
assert utils.create_dir(dir2) is None
os.rmdir(dir2)
assert utils.create_dir(dir3) is None
os.makedirs = fakeos.makedirs2
self.assertRaises(Exception, utils.create_dir, dir2)
def test_create_dir(self, monkeypatch):
dir1 = '/tmp'
dir2 = '/tmp/testnoexistent1234'
dir3 = '~'
fakeos = Os()
assert create_dir(dir1) is None
assert create_dir(dir2) is None
os.rmdir(dir2)
assert create_dir(dir3) is None
monkeypatch.setattr(os, 'makedirs', fakeos.makedirs2)
pytest.raises(Exception, create_dir, dir2)
def tar_incremental(
tar_cmd, backup_opt_dict, curr_tar_meta, remote_manifest_meta=None):
'''
Check if the backup already exist in swift. If the backup already
exists, the related meta data and the tar incremental meta file will be
downloaded. According to the meta data content, new options will be
provided for the next meta data upload to swift and the existing tar meta
file will be used in the current incremental backup. Also the level
options will be checked and updated respectively
'''
if not tar_cmd or not backup_opt_dict:
logging.error('[*] Error: tar_incremental, please provide tar_cmd \
and backup options')
raise ValueError
if not remote_manifest_meta:
remote_manifest_meta = dict()
# If returned object from check_backup is not a dict, the backup
# is considered at first run, so a backup level 0 will be executed
curr_backup_level = remote_manifest_meta.get(
'x-object-meta-backup-current-level', '0')
tar_meta = remote_manifest_meta.get(
'x-object-meta-tar-meta-obj-name')
tar_cmd_level = '--level={0} '.format(curr_backup_level)
# Write the tar meta data file in ~/.freezer. It will be
# removed later on. If ~/.freezer does not exists it will be created'.
create_dir(backup_opt_dict.workdir)
curr_tar_meta = '{0}/{1}'.format(
backup_opt_dict.workdir, curr_tar_meta)
tar_cmd_incr = ' --listed-incremental={0} '.format(curr_tar_meta)
if tar_meta:
# If tar meta data file is available, download it and use it
# as for tar incremental backup. Afte this, the
# remote_manifest_meta['x-object-meta-tar-meta-obj-name'] will be
# update with the current tar meta data name and uploaded again
tar_cmd_incr = ' --listed-incremental={0}/{1} '.format(
backup_opt_dict.workdir, tar_meta)
tar_meta_abs = "{0}/{1}".format(backup_opt_dict.workdir, tar_meta)
try:
object_to_file(
backup_opt_dict, tar_meta_abs)
except Exception:
logging.warning(
'[*] Tar metadata {0} not found. Executing level 0 backup\
'.format(tar_meta))
tar_cmd = ' {0} {1} {2} '.format(tar_cmd, tar_cmd_level, tar_cmd_incr)
return tar_cmd, backup_opt_dict, remote_manifest_meta
def main():
doers = _get_doers(shell)
doers.update(_get_doers(utils))
possible_actions = doers.keys() + ['start', 'stop', 'status']
args = arguments.get_args(possible_actions)
if args.action is None:
print "No action"
sys.exit(1)
apiclient = None
if args.no_api is False:
os_options = arguments.OpenstackOptions(args, os.environ)
if args.debug:
print os_options
apiclient = client.Client(username=os_options.username,
password=os_options.password,
tenant_name=os_options.tenant_name,
endpoint=os_options.endpoint,
auth_url=os_options.auth_url)
if args.client_id:
apiclient.client_id = args.client_id
if args.action in doers:
try:
return doers[args.action](apiclient, args)
except Exception as e:
print ('ERROR {0}'.format(e))
return 1
create_dir(args.jobs_dir, do_log=False)
freezer_scheduler = FreezerScheduler(apiclient=apiclient,
interval=int(args.interval),
job_path=args.jobs_dir)
daemon = Daemon(daemonizable=freezer_scheduler)
if args.action == 'start':
daemon.start(log_file=args.log_file)
elif args.action == 'stop':
daemon.stop()
elif args.action == 'reload':
daemon.reload()
elif args.action == 'status':
daemon.status()
return os.EX_OK
def tar_incremental(tar_cmd,
backup_opt_dict,
curr_tar_meta,
remote_manifest_meta=None):
"""
Check if the backup already exist in swift. If the backup already
exists, the related meta data and the tar incremental meta file will be
downloaded. According to the meta data content, new options will be
provided for the next meta data upload to swift and the existing tar meta
file will be used in the current incremental backup. Also the level
options will be checked and updated respectively
"""
if not tar_cmd or not backup_opt_dict:
logging.error(('[*] Error: tar_incremental, please provide tar_cmd '
'and backup options'))
raise ValueError
if not remote_manifest_meta:
remote_manifest_meta = dict()
# If returned object from check_backup is not a dict, the backup
# is considered at first run, so a backup level 0 will be executed
curr_backup_level = remote_manifest_meta.get(
'x-object-meta-backup-current-level', '0')
tar_meta = remote_manifest_meta.get('x-object-meta-tar-meta-obj-name')
tar_cmd_level = '--level={0} '.format(curr_backup_level)
# Write the tar meta data file in ~/.freezer. It will be
# removed later on. If ~/.freezer does not exists it will be created'.
create_dir(backup_opt_dict.workdir)
curr_tar_meta = '{0}/{1}'.format(backup_opt_dict.workdir, curr_tar_meta)
tar_cmd_incr = ' --listed-incremental={0} '.format(curr_tar_meta)
if tar_meta:
# If tar meta data file is available, download it and use it
# as for tar incremental backup. Afte this, the
# remote_manifest_meta['x-object-meta-tar-meta-obj-name'] will be
# update with the current tar meta data name and uploaded again
tar_cmd_incr = ' --listed-incremental={0}/{1} '.format(
backup_opt_dict.workdir, tar_meta)
tar_meta_abs = "{0}/{1}".format(backup_opt_dict.workdir, tar_meta)
try:
object_to_file(backup_opt_dict, tar_meta_abs)
except Exception:
logging.warning(
'[*] Tar metadata {0} not found. Executing level 0 backup\
'.format(tar_meta))
tar_cmd = ' {0} {1} {2} '.format(tar_cmd, tar_cmd_level, tar_cmd_incr)
return tar_cmd, backup_opt_dict, remote_manifest_meta
def test_backup_file(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), "file_1")
os.chdir(files_dir)
storage.backup(files_dir + "/file_1", "file_backup", builder)
for path in os.listdir(files_dir):
os.remove(files_dir + "/" + path)
assert not os.listdir(files_dir)
utils.create_dir(files_dir)
backup = storage.get_backups()[0]
builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir)
storage.restore(backup, files_dir, builder, 0)
files = os.listdir(files_dir)
assert len(files) == 1
def download_meta_file(self, backup):
"""
:type backup: freezer.storage.Backup
:param backup:
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return utils.path_join(self.work_dir, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
zero_backup = self._zero_backup_dir(backup)
from_path = utils.path_join(zero_backup, meta_backup.tar())
to_path = utils.path_join(self.work_dir, meta_backup.tar())
if os.path.exists(to_path):
os.remove(to_path)
self.get_file(from_path, to_path)
return to_path
def start(self, log_file=None):
"""Initialize freezer-scheduler instance inside a windows service
"""
setup_logging(log_file)
utils.create_dir(self.home)
if self.insecure:
os.environ['SERVICE_INSECURE'] = 'True'
# send arguments info to the windows service
os.environ['SERVICE_JOB_PATH'] = self.job_path
os.environ['SERVICE_INTERVAL'] = str(self.interval)
winutils.save_environment(self.home)
print('Freezer Service is starting')
win32serviceutil.StartService(self.service_name)
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
format=('%(asctime)s %(name)s %(levelname)s %(message)s'))
# filter out some annoying messages
# not the best position for this code
log_filter = NoLogFilter()
logging.getLogger("apscheduler.scheduler").\
addFilter(log_filter)
logging.getLogger("apscheduler.executors.default").\
addFilter(log_filter)
logging.getLogger("requests.packages.urllib3.connectionpool").\
addFilter(log_filter)
return expanded_file_name
def download_meta_file(self, backup):
"""
:type backup: freezer.storage.Backup
:param backup:
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return "{0}{1}{2}".format(self.work_dir, os.sep, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
zero_backup = self._zero_backup_dir(backup)
from_path = "{0}{1}{2}".format(zero_backup, os.sep, meta_backup.tar())
to_path = "{0}{1}{2}".format(self.work_dir, os.sep, meta_backup.tar())
if backup.level != 0:
if os.path.exists(to_path):
os.remove(to_path)
self.ftp.get(from_path, to_path)
return to_path
def download_meta_file(self, backup):
"""
Downloads meta_data to work_dir of previous backup.
:type backup: freezer.storage.base.Backup
:param backup: A backup or increment. Current backup is incremental,
that means we should download tar_meta for detection new files and
changes. If backup.tar_meta is false, raise Exception
:return:
"""
utils.create_dir(self.work_dir)
if backup.level == 0:
return utils.path_join(self.work_dir, backup.tar())
meta_backup = backup.full_backup.increments[backup.level - 1]
if not meta_backup.tar_meta:
raise ValueError('Latest update have no tar_meta')
to_path = utils.path_join(self.work_dir, meta_backup.tar())
if os.path.exists(to_path):
os.remove(to_path)
meta_backup.storage.get_file(
meta_backup.storage.meta_file_abs_path(meta_backup), to_path)
return to_path
def backup(self, path, hostname_backup_name, tar_builder,
parent_backup=None):
"""
Backup path
storage_dir/backup_name/timestamp/backup_name_timestamps_level
:param path:
:param hostname_backup_name:
:param tar_builder:
:type tar_builder: freezer.tar.TarCommandBuilder
:param parent_backup:
:type parent_backup: freezer.storage.Backup
:return:
"""
new_backup = self._create_backup(hostname_backup_name, parent_backup)
host_backups = self._backup_dir(new_backup)
utils.create_dir(host_backups)
if parent_backup:
zero_backup = self._zero_backup_dir(parent_backup.parent)
else:
zero_backup = self._zero_backup_dir(new_backup)
utils.create_dir(zero_backup)
tar_builder.set_output_file("{0}/{1}".format(zero_backup,
new_backup.repr()))
tar_incremental = "{0}/{1}".format(zero_backup, new_backup.tar())
if parent_backup:
shutil.copyfile("{0}/{1}".format(
zero_backup, parent_backup.tar()), tar_incremental)
tar_builder.set_listed_incremental(tar_incremental)
logging.info('[*] Changing current working directory to: {0}'
.format(path))
logging.info('[*] Backup started for: {0}'.format(path))
subprocess.check_output(tar_builder.build(), shell=True)
def lvm_eval(backup_opt_dict):
"""
Evaluate if the backup must be executed using lvm snapshot
or just directly on the plain filesystem. If no lvm options are specified
the backup will be executed directly on the file system and without
use lvm snapshot. If one of the lvm options are set, then the lvm snap
will be used to execute backup. This mean all the required options
must be set accordingly
"""
required_list = [
backup_opt_dict.lvm_volgroup,
backup_opt_dict.lvm_srcvol,
backup_opt_dict.lvm_dirmount]
if not validate_all_args(required_list):
logging.warning('[*] Required lvm options not set. The backup will \
execute without lvm snapshot.')
return False
# Create lvm_dirmount dir if it doesn't exists and write action in logs
create_dir(backup_opt_dict.lvm_dirmount)
return True
def test_incremental_restore(self, tmpdir):
backup_dir, files_dir = self.create_dirs(tmpdir)
storage = local.LocalStorage(backup_dir)
builder = tar.TarCommandBuilder(commons.tar_path(), ".")
os.chdir(files_dir)
storage.backup(files_dir, "file_backup", builder)
backups = storage.get_backups()
assert len(backups) == 1
backup = backups[0]
self.create_content(files_dir, "file_2", "foo\n")
storage.backup(files_dir, "file_backup", builder, backup)
for path in os.listdir(files_dir):
os.remove(files_dir + "/" + path)
assert not os.listdir(files_dir)
utils.create_dir(files_dir)
backup = storage.get_backups()[0]
builder = tar.TarCommandRestoreBuilder(commons.tar_path(), files_dir)
storage.restore(backup, files_dir, builder, 1)
files = os.listdir(files_dir)
assert len(files) == 2
with open(files_dir + "/file_1", "r") as file_1:
assert self.HELLO == file_1.read()
with open(files_dir + "/file_2", "r") as file_2:
assert "foo\n" == file_2.read()
def lvm_snap(backup_opt_dict):
"""
Checks the provided parameters and create the lvm snapshot if requested
The path_to_backup might be adjusted in case the user requested
a lvm snapshot without specifying an exact path for the snapshot
(lvm_auto_snap).
The assumption in this case is that the user wants to use the lvm snapshot
capability to backup the specified filesystem path, leaving out all
the rest of the parameters which will guessed and set by freezer.
if a snapshot is requested using the --snapshot flag, but lvm_auto_snap
is not provided, then path_to_backup is supposed to be the path to backup
*before* any information about the snapshot is added and will be
adjusted.
:param backup_opt_dict: the configuration dict
:return: True if the snapshot has been taken, False otherwise
"""
if backup_opt_dict.snapshot:
if not backup_opt_dict.lvm_auto_snap:
# 1) the provided path_to_backup has the meaning of
# the lvm_auto_snap and is therefore copied into it
# 2) the correct value of path_to_backup, which takes into
# consideration the snapshot mount-point, is cleared
# and will be calculated by freezer
backup_opt_dict.lvm_auto_snap =\
backup_opt_dict.path_to_backup
backup_opt_dict.path_to_backup = ''
if not backup_opt_dict.lvm_snapname:
backup_opt_dict.lvm_snapname = \
"{0}_{1}".format(freezer_config.DEFAULT_LVM_SNAP_BASENAME,
uuid.uuid4().hex)
if backup_opt_dict.lvm_auto_snap:
# adjust/check lvm parameters according to provided lvm_auto_snap
lvm_info = get_lvm_info(backup_opt_dict.lvm_auto_snap)
if not backup_opt_dict.lvm_volgroup:
backup_opt_dict.lvm_volgroup = lvm_info['volgroup']
if not backup_opt_dict.lvm_srcvol:
backup_opt_dict.lvm_srcvol = lvm_info['srcvol']
if not backup_opt_dict.lvm_dirmount:
backup_opt_dict.lvm_dirmount = \
"{0}_{1}".format(freezer_config.DEFAULT_LVM_MOUNT_BASENAME,
uuid.uuid4().hex)
path_to_backup = os.path.join(backup_opt_dict.lvm_dirmount,
lvm_info['snap_path'])
if backup_opt_dict.path_to_backup:
# path_to_backup is user-provided, check if consistent
if backup_opt_dict.path_to_backup != path_to_backup:
raise Exception('Path to backup mismatch. '
'provided: {0}, should be LVM-mounted: {1}'.
format(backup_opt_dict.path_to_backup,
path_to_backup))
else:
# path_to_backup not provided: use the one calculated above
backup_opt_dict.path_to_backup = path_to_backup
if not validate_lvm_params(backup_opt_dict):
logging.info('[*] No LVM requested/configured')
return False
utils.create_dir(backup_opt_dict.lvm_dirmount)
lvm_create_command = (
'{0} --size {1} --snapshot --permission {2} '
'--name {3} {4}'.format(
utils.find_executable('lvcreate'),
backup_opt_dict.lvm_snapsize,
('r' if backup_opt_dict.lvm_snapperm == 'ro'
else backup_opt_dict.lvm_snapperm),
backup_opt_dict.lvm_snapname,
backup_opt_dict.lvm_srcvol))
lvm_process = subprocess.Popen(
lvm_create_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
(lvm_out, lvm_err) = lvm_process.communicate()
if lvm_process.returncode:
raise Exception('lvm snapshot creation error: {0}'.format(lvm_err))
logging.debug('[*] {0}'.format(lvm_out))
logging.warning('[*] Logical volume "{0}" created'.
format(backup_opt_dict.lvm_snapname))
# Guess the file system of the provided source volume and st mount
# options accordingly
filesys_type = utils.get_vol_fs_type(backup_opt_dict.lvm_srcvol)
mount_options = '-o {}'.format(backup_opt_dict.lvm_snapperm)
if 'xfs' == filesys_type:
mount_options = ' -onouuid '
# Mount the newly created snapshot to dir_mount
abs_snap_name = '/dev/{0}/{1}'.format(
backup_opt_dict.lvm_volgroup,
backup_opt_dict.lvm_snapname)
mount_command = '{0} {1} {2} {3}'.format(
utils.find_executable('mount'),
mount_options,
abs_snap_name,
backup_opt_dict.lvm_dirmount)
mount_process = subprocess.Popen(
mount_command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
executable=utils.find_executable('bash'))
mount_err = mount_process.communicate()[1]
if 'already mounted' in mount_err:
logging.warning('[*] Volume {0} already mounted on {1}\
'.format(abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
if mount_err:
logging.error("[*] Snapshot mount error. Removing snapshot")
lvm_snap_remove(backup_opt_dict)
raise Exception('lvm snapshot mounting error: {0}'.format(mount_err))
else:
logging.warning(
'[*] Volume {0} succesfully mounted on {1}'.format(
abs_snap_name, backup_opt_dict.lvm_dirmount))
return True
def prepare(self):
utils.create_dir(self.storage_directory)
