def setup_temp_conf_dir_for_pool(pool=''):
""" setup temp conf dir for rsnap confs
if provided, use, otherwise make temp dir
"""
if not pool:
pool = settings.POOL
logger = logs.get_logger()
if settings.TEMP_CONF_DIR:
if os.path.isdir(settings.TEMP_CONF_DIR):
logger.info('using temp conf dir %s' % settings.TEMP_CONF_DIR)
else:
try:
setup_dir(settings.TEMP_CONF_DIR)
except IOError as e:
logger.error(
'Cannot create conf temp dir (or intermediate dirs) from' +
' setting %s with error %s' % (settings.TEMP_CONF_DIR, e))
raise
else:
try:
logger.info('making a tempdir for rsnap confs')
temp_conf_dir = make_empty_tempdir(
prefix=settings.TEMP_CONF_DIR_PREFIX)
# store this in global settings
settings.TEMP_CONF_DIR = temp_conf_dir
except IOError as e:
logger.error('cannot create conf temp dir with error %s' % e)
raise
# now make for pool
pool_conf_dir = "%s/%s" % (settings.TEMP_CONF_DIR, pool)
setup_dir(pool_conf_dir)
return settings.TEMP_CONF_DIR
def remove_snap_status_file(snap_date,
cephhost='',
snap_status_file_path='',
noop=''):
logger = logs.get_logger()
if not cephhost:
cephhost = settings.CEPH_HOST
if not snap_status_file_path:
snap_status_file_path = settings.SNAP_STATUS_FILE_PATH
if not noop:
noop = settings.NOOP
REMOVE_SNAP_STATUS_FILE_COMMAND = ('rm -fv %s/%s' %
(snap_status_file_path, snap_date))
logger.info('removing snap status file on ceph host with command %s' %
REMOVE_SNAP_STATUS_FILE_COMMAND)
if noop:
logger.info('would have run %s' % REMOVE_SNAP_STATUS_FILE_COMMAND)
remove_snap_status_file_result = 'noop'
else:
remove_snap_status_file_result = sh.ssh(
cephhost, REMOVE_SNAP_STATUS_FILE_COMMAND).strip('\n')
# TODO handle some errors gracefully here
logger.info("done removing snap status file: %s" %
remove_snap_status_file_result)
return True
def remove_empty_dir(directory):
""" remove a directory if it's empty
"""
logger = logs.get_logger()
if settings.NOOP:
logger.info('NOOP: would have removed %s' % directory)
else:
logger.info('removing %s' % directory)
os.rmdir(directory)
def remove_qcow(image,
pool='',
cephhost='',
cephuser='',
cephcluster='',
snap_naming_date_format='',
snap_date='',
snap='',
noop=None):
""" ssh to ceph node and remove a qcow from path
qcow_temp_path/pool/imagename.qcow2
"""
logger = logs.get_logger()
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
if not snap:
snap = get_snapdate(snap_naming_date_format=snap_naming_date_format,
snap_date=snap_date)
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not cephuser:
cephuser = settings.CEPH_USER
# TODO use ceph cluster in path naming
if not cephcluster:
cephcluster = settings.CEPH_CLUSTER
if not noop:
noop = settings.NOOP
temp_qcow_file = ("%s/%s/%s@%s.qcow2" %
(settings.QCOW_TEMP_PATH, settings.POOL, image, snap))
logger.info("deleting temp qcow from path %s on ceph host %s" %
(temp_qcow_file, cephhost))
SSH_RM_QCOW_COMMAND = 'rm %s' % temp_qcow_file
try:
if settings.NOOP:
logger.info('NOOP: would have removed temp qcow for image %s from'
' ceph host %s with command %s' %
(image, cephhost, SSH_RM_QCOW_COMMAND))
else:
sh.ssh(cephhost, SSH_RM_QCOW_COMMAND)
except sh.ErrorReturnCode as e:
logger.error('error removing temp qcow %s with error from ssh:' %
temp_qcow_file)
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error removing temp qcow %s' % temp_qcow_file)
logger.exception(e)
raise
logger.info("successfully removed qcow for %s" % image)
return True
def remove_conf(image, pool=''):
if not pool:
pool = settings.POOL
# get logger we setup earlier
logger = logs.get_logger()
if settings.NOOP:
logger.info('NOOP: would have removed conf file %s/%s/%s.conf' %
(settings.TEMP_CONF_DIR, pool, image))
else:
logger.info('removing temp rsnap conf file for image %s' % image)
os.remove('%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image))
def setup_log_dirs_for_pool(pool=''):
""" wrapper of the above to setup log dirs
"""
logger = logs.get_logger()
if not pool:
pool = settings.POOL
dirs = [
settings.LOG_BASE_PATH,
"%s/rsnap" % settings.LOG_BASE_PATH,
"%s/rsnap/%s" % (settings.LOG_BASE_PATH, pool),
]
for directory in dirs:
setup_dir(directory, perms=0o755)
def make_empty_tempdir(prefix=''):
""" make an empty tempdir
"""
logger = logs.get_logger()
if not prefix:
prefix = 'empty_'
if settings.NOOP:
logger.info('NOOP: would have made a tempdir with prefix %s' % prefix)
return 'noop_fake_empty_path_with_prefix_%s' % prefix
else:
logger.info('creating tempdir with prefix %s' % prefix)
empty_tempdir = tempfile.mkdtemp(prefix=prefix)
return empty_tempdir
def setup_dir(directory, perms=0o700):
logger = logs.get_logger()
if not os.path.isdir(directory):
# make dir and preceeding dirs if necessary
if settings.NOOP:
logger.info('NOOP: would have run makedirs on path %s' % directory)
else:
logger.info('creating directory %o %s' % (perms, directory))
os.makedirs(directory, perms)
else:
logger.info('directory %s already exists, so using it' % directory)
# still need to check perms
check_set_dir_perms(directory, perms)
def check_snap_status_file(cephhost='', snap_status_file_path=''):
logger = logs.get_logger()
if not cephhost:
cephhost = settings.CEPH_HOST
if not snap_status_file_path:
snap_status_file_path = settings.SNAP_STATUS_FILE_PATH
CHECK_SNAP_STATUS_DIR_COMMAND = ('ls -t %s/*' % snap_status_file_path)
logger.info('checking snap status directory %s on ceph host' %
snap_status_file_path)
try:
snap_status_dir_result = sh.ssh(
cephhost, CHECK_SNAP_STATUS_DIR_COMMAND).strip('\n')
except sh.ErrorReturnCode as e:
if e.exit_code == 2:
raise exceptions.NoSnapStatusFilesFoundError(
cephhost=cephhost, status_dir=snap_status_file_path, e=e)
else:
raise
logger.debug("found: %s" % snap_status_dir_result)
snap_dates = [
snap_status_file.split('/')[-1]
for snap_status_file in snap_status_dir_result.split('\n')
]
logger.debug("snap dates:")
logger.debug(snap_dates)
snap_date = snap_dates[0]
logger.debug("checking newest snap_date %s" % snap_date)
try:
result = check_formatted_snap_date(snap_date=snap_date)
except exceptions.SnapDateNotValidDateError as e:
raise
# if we're here it was a valid date and matches format
# remove the rest of them that match
for old_snap_date in snap_dates[1:]:
logger.warn(
"found old snap_date files- checking and removing if valid dates")
try:
logger.debug('checking old snap date %s' % old_snap_date)
check_formatted_snap_date(snap_date=old_snap_date)
except exceptions.SnapDateNotValidDateError as e:
e.log(warn=True)
continue
except exceptions.SnapDateFormatMismatchError as e:
e.log(warn=True)
continue
# if here then it's a valid date
logger.warning('removing old snap_date status file %s because we have'
' a newer one %s' % (old_snap_date, snap_date))
remove_snap_status_file(snap_date=old_snap_date)
logger.info('using snap_date %s found from queue on ceph host' % snap_date)
return snap_date
def get_rbd_size(image,
snap='',
pool='',
cephhost='',
cephuser='',
cephcluster='',
snap_naming_date_format='',
snap_date=''):
"""ssh to ceph node check the size of this image@snap
"""
logger = logs.get_logger()
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
if not snap:
snap = get_snapdate(snap_naming_date_format=snap_naming_date_format,
snap_date=snap_date)
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not cephuser:
cephuser = settings.CEPH_USER
if not cephcluster:
cephcluster = settings.CEPH_CLUSTER
rbd_image_string = "%s/%s@%s" % (pool, image, snap)
RBD_COMMAND = ('rbd du %s --user=%s --cluster=%s --format=json' %
(rbd_image_string, cephuser, cephcluster))
logger.info('getting rbd size from ceph host %s with command %s' %
(cephhost, RBD_COMMAND))
try:
rbd_du_result = sh.ssh(cephhost, RBD_COMMAND)
rbd_image_used_size = (json.loads(
rbd_du_result.stdout)['images'][0]['used_size'])
rbd_image_provisioned_size = (json.loads(
rbd_du_result.stdout)['images'][0]['provisioned_size'])
# using provisioned_size as these are snaps and space could be on the
# parent
return rbd_image_provisioned_size
except sh.ErrorReturnCode as e:
logger.error('error getting rbd size for %s, output from ssh:' %
rbd_image_string)
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error getting rbd size for %s' % rbd_image_string)
logger.exception(e)
raise
def check_set_dir_perms(directory, perms=0o700):
logger = logs.get_logger()
desired_mode = oct(perms)[-3:]
if settings.NOOP:
logger.info('NOOP: would have verified that permissions on %s are %s' %
(directory, desired_mode))
else:
dir_stat = os.stat(directory)
current_mode = oct(dir_stat.st_mode)[-3:]
if current_mode != desired_mode:
logger.warning(
'perms not correct on %s: currently %s should be %s,'
'fixing' % (directory, current_mode, desired_mode))
os.chmod(directory, perms)
logger.info('perms now correctly set to %s on %s' %
(desired_mode, directory))
def setup_backup_dirs_for_pool(pool='', dirs=''):
""" wrapper of the above to setup backup dirs
"""
logger = logs.get_logger()
if not pool:
pool = settings.POOL
if not dirs:
dirs = [
settings.BACKUP_BASE_PATH,
"%s/%s" % (settings.BACKUP_BASE_PATH, pool),
]
# TODO support multiple pools
# for pool in settings.POOLS
# dirs.append(pool)
for directory in dirs:
logger.info('setting up backup dir: %s' % directory)
setup_dir(directory)
def remove_temp_conf_dir():
""" remove temp conf dirs
"""
logger = logs.get_logger()
if not settings.KEEPCONF:
logger.info("removing temp conf dir %s" % settings.TEMP_CONF_DIR)
try:
if settings.NOOP:
logger.info('NOOP: would have removed temp conf dirs %s/%s and'
' %s' % (settings.TEMP_CONF_DIR, settings.POOL,
settings.TEMP_CONF_DIR))
else:
# TODO all pools
os.rmdir("%s/%s" % (settings.TEMP_CONF_DIR, settings.POOL))
os.rmdir(settings.TEMP_CONF_DIR)
except (IOError, OSError) as e:
logger.warning("unable to remove temp conf dir with error %s" % e)
def get_snapdate(snap_naming_date_format='', snap_date=''):
"""get todays date in iso format, this can run on either node
"""
logger = logs.get_logger()
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
try:
converted_snap_date = sh.date('+%s' % snap_naming_date_format,
date=snap_date).strip('\n')
except sh.ErrorReturnCode as e:
if e.exit_code == 1:
raise (exceptions.SnapDateNotValidDateError(
snap_date=snap_date, date_format=snap_naming_date_format, e=e))
else:
raise
return converted_snap_date
def validate_settings_strings():
""" check all settings strings to make sure they are only safe chars
if not fail run
"""
logger = logs.get_logger()
logger.debug('checking settings strings to ensure they only contain safe'
' chars: %s' % settings.STRING_SAFE_CHAR_RE)
# check strings are safe
current_settings = get_current_settings()
for key in current_settings:
if key in settings.ADDITIONAL_SAFE_CHARS:
additional_safe_chars = settings.ADDITIONAL_SAFE_CHARS[key]
else:
additional_safe_chars = ''
if key in ['IMAGE_RE']:
# these are allowed to have weird characters
# also this is only used in this script as a RE
continue
value = current_settings[key]
if type(value) in [bool, int]:
# don't compare these to an RE
continue
try:
if key == 'POOLS':
logger.debug('checking pools string %s' % value)
pools_arr = value.split(',')
for pool in pools_arr:
logger.debug('checking string for pool %s' % pool)
validate_string(
pool, additional_safe_chars=additional_safe_chars)
# if here then they all validated
continue
if validate_string(value,
additional_safe_chars=additional_safe_chars):
continue
except NameError as e:
# bad character in a string, fail run
raise NameError('disallowed character in setting: %s'
' error %s' % (key, e))
except Exception as e:
raise
logger.debug('all settings strings ok')
def check_snap(image,
snap='',
pool='',
cephhost='',
cephuser='',
cephcluster='',
snap_naming_date_format='',
snap_date=''):
""" ssh to ceph host and check for a snapshot
"""
logger = logs.get_logger()
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
if not snap:
snap = get_snapdate(snap_naming_date_format=snap_naming_date_format,
snap_date=snap_date)
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not cephuser:
cephuser = settings.CEPH_USER
if not cephcluster:
cephcluster = settings.CEPH_CLUSTER
RBD_CHECK_SNAP_COMMAND = ('rbd info %s/%s@%s --id=%s --cluster=%s' %
(pool, image, snap, cephuser, cephcluster))
logger.info('checking for snap with command %s' % RBD_CHECK_SNAP_COMMAND)
try:
rbd_check_result = sh.ssh(cephhost, RBD_CHECK_SNAP_COMMAND)
except sh.ErrorReturnCode as e:
# this just means no snap found, log but don't raise
logger.warning('no snap found for image %s' % image)
# return false we didn't find one
return False
except Exception as e:
logger.info('error getting list of images from ceph node')
logger.exception(e)
raise
# if here then rbd snap found, so return True
return True
def get_names_on_dest(pool=''):
logger = logs.get_logger()
if not pool:
pool = settings.POOL
backup_base_path = settings.BACKUP_BASE_PATH
# get logger we setup earlier
logger = logging.getLogger('ceph_rsnapshot')
backup_path = "%s/%s" % (backup_base_path, pool)
try:
names_on_dest = os.listdir(backup_path)
except (IOError, OSError) as e:
if settings.NOOP:
# this will fail if noop and the dir doesn't exist, so
# fake nothing there and move on
logger.info('NOOP: would have listed vms in directory %s' %
backup_path)
return []
logger.error(e)
raise NameError('get_names_on_dest failed with error %s' % e)
# FIXME cehck error
return names_on_dest
def get_freespace(path=''):
"""ssh to ceph node and get freespace for a path
if not specified, use settings.QCOW_TEMP_PATH/settings.POOL
"""
logger = logs.get_logger()
if not path:
path = "%s/%s" % (settings.QCOW_TEMP_PATH, settings.POOL)
DF_COMMAND = ("LANG='' LC_CTYPE='' df -P %s | grep / | awk '{print $4}';"
" ( exit ${PIPESTATUS[0]} )" % path)
try:
free_space_kb = sh.ssh(settings.CEPH_HOST, DF_COMMAND)
free_space_bytes = int(free_space_kb.stdout) * 1024
return free_space_bytes
except sh.ErrorReturnCode as e:
logger.error('error getting free space on ceph node for %s' % path)
logger.exception(e.stderr)
# pass it up
raise
except Exception as e:
logger.error('error getting free space on ceph node')
logger.exception(e)
# pass it up
raise
def check_qcow_temp_path_empty_for_pool(cephhost='',
qcowtemppath='',
pool='',
noop=None):
logger = logs.get_logger()
if not cephhost:
cephhost = settings.CEPH_HOST
if not qcowtemppath:
qcowtemppath = settings.QCOW_TEMP_PATH
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not noop:
noop = settings.NOOP
if noop:
# this dir might not exist yet so just say it's good and move on
return True
temp_path = '%s/%s' % (qcowtemppath, pool)
logger.info('checking qcow temp export path %s is empty on ceph host'
' %s' % (temp_path, cephhost))
LS_COMMAND = 'ls -a %s' % temp_path
try:
ls_result = sh.ssh(cephhost, LS_COMMAND)
if ls_result == EMPTY_DIR_LS_RESULT:
return True
else:
logger.error('ERROR: temp qcow export directory %s not empty: %s',
temp_path, ls_result)
except sh.ErrorReturnCode as e:
logger.error('error checking temp qcow export directory')
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error checking temp qcow export directory')
logger.exception(e)
raise
def rsnap_image_sh(image, pool=''):
logger = logs.get_logger()
if not pool:
pool = settings.POOL
# TODO check free space before rsnapping
logger.info("rsnapping %s" % image)
rsnap_conf_file = '%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image)
if settings.NOOP:
logger.info(
'NOOP: would have rsnapshotted image from conf file '
'%s/%s/%s.conf for retain interval %s ' %
(settings.TEMP_CONF_DIR, pool, image, settings.RETAIN_INTERVAL))
# set this False so it's clear this wasn't successful as it was a noop
rsnap_ok = False
else:
try:
ts = time.time()
rsnap_result = sh.rsnapshot('-c', rsnap_conf_file,
settings.RETAIN_INTERVAL)
tf = time.time()
elapsed_time = tf - ts
elapsed_time_ms = elapsed_time * 10**3
rsnap_ok = True
logger.info("rsnap successful for image %s in %sms" %
(image, elapsed_time_ms))
if rsnap_result.stdout.strip("\n"):
logger.info("stdout from rsnap:\n" +
rsnap_result.stdout.strip("\n"))
except sh.ErrorReturnCode as e:
logger.error("failed to rsnap %s with code %s" %
(image, e.exit_code))
# TODO move log formatting and writing to a function
logger.error("stdout from rsnap:\n" + e.stdout.strip("\n"))
logger.error("stderr from rsnap:\n" + e.stderr.strip("\n"))
rsnap_ok = False
return rsnap_ok
def rsnap_image(image, pool='', template=None):
if not pool:
pool = settings.POOL
temp_path = settings.QCOW_TEMP_PATH
extra_args = settings.EXTRA_ARGS
conf_base_path = settings.TEMP_CONF_DIR
backup_base_path = settings.BACKUP_BASE_PATH
# get logger we setup earlier
logger = logs.get_logger()
logger.info('working on image %s in pool %s' % (image, pool))
# setup flags
qcow_temp_path_empty = False
export_qcow_ok = False
rsnap_ok = False
remove_qcow_ok = False
# only reopen if we haven't pulled this yet - ie, are we part of a pool run
if not template:
template = templates.get_template()
# create the temp conf file
conf_file = templates.write_conf(image, pool=pool, template=template)
logger.info(conf_file)
# make sure temp qcow dir is empty
try:
if dirs.check_qcow_temp_path_empty_for_pool(pool=pool):
qcow_temp_path_empty = True
except Exception as e:
# if it's not empty, fail this image
logger.error('qcow temp path not empty, failing this image')
logger.exception(e)
qcow_temp_path_empty = False
# ssh to source and export temp qcow of this image
if qcow_temp_path_empty:
try:
ceph.export_qcow(image, pool=pool)
export_qcow_ok = True
except NameError as e:
# probably not enough space. set to false and try to go and remove this
# one, or go to next image in case it was temporary
logger.error('error from export qcow: %s' % e)
export_qcow_ok = False
except Exception as e:
logger.error('error from export qcow')
logger.exception(e)
export_qcow_ok = False
# if exported ok, then rsnap this image
if export_qcow_ok:
try:
rsnap_ok = rsnap_image_sh(image, pool=pool)
except Exception as e:
# TODO
logger.error('error with rsnapping image %s' % image)
else:
logger.error(
"skipping rsnap of image %s because export to qcow failed" % image)
# either way remove the temp qcow
logger.info("removing temp qcow for %s" % image)
try:
remove_qcow_ok = ceph.remove_qcow(image, pool=pool)
except Exception as e:
logger.error(
'error removing qcow. will continue to next image anyways,'
' note that we check for free space so wont entirely fill disk if they'
' all fail')
# either way remove the temp conf file
# unless flag to keep it for debug
if not settings.KEEPCONF:
templates.remove_conf(image, pool=pool)
if export_qcow_ok and rsnap_ok and remove_qcow_ok:
successful = True
else:
successful = False
# return a blob with the details
return ({
'image': image,
'pool': pool,
'successful': successful,
'status': {
'qcow_temp_path_empty': qcow_temp_path_empty,
'export_qcow_ok': export_qcow_ok,
'rsnap_ok': rsnap_ok,
'remove_qcow_ok': remove_qcow_ok
}
})
def setup_qcow_temp_path(pool='', cephhost='', qcowtemppath='', noop=None):
""" ssh to ceph node and check or make temp qcow export path
"""
logger = logs.get_logger()
if not qcowtemppath:
qcowtemppath = settings.QCOW_TEMP_PATH
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not noop:
noop = settings.NOOP
temp_path = '%s/%s' % (qcowtemppath, pool)
logger.info('making qcow temp export path %s on ceph host %s' %
(temp_path, cephhost))
LS_COMMAND = 'ls %s' % temp_path
MKDIR_COMMAND = 'mkdir -p %s' % temp_path
CHMOD_COMMAND = 'LANG=' ' LC_CTYPE=' ' chmod 700 %s' % temp_path
try:
ls_result = sh.ssh(cephhost, LS_COMMAND)
except sh.ErrorReturnCode as e:
if e.exit_code == 2:
# ls returns 2 for no such dir, this is OK, just make it
try:
if noop:
logger.info('NOOP: would have made qcow temp path %s' %
temp_path)
else:
sh.ssh(cephhost, MKDIR_COMMAND)
sh.ssh(cephhost, CHMOD_COMMAND)
except sh.ErrorReturnCode as e:
logger.error('error making or chmodding qcow temp dir:')
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error making or chmodding qcow temp dir')
logger.exception(e)
raise
else:
logger.error('error checking temp qcow export directory')
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error checking temp qcow export directory')
logger.exception(e)
raise
logger.info('using qcow temp path: %s' % temp_path)
# now just to be safe verify perms on it are 700
try:
if settings.NOOP:
logger.info(
'NOOP: would have chmodded qcow temp path with command:'
' %s' % CHMOD_COMMAND)
else:
sh.ssh(cephhost, CHMOD_COMMAND)
except sh.ErrorReturnCode as e:
logger.error('error chmodding qcow temp dir:')
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error chmodding qcow temp dir')
logger.exception(e)
raise
def rsnap_pool(pool):
# get values from settings
host = settings.CEPH_HOST
# get logger we setup earlier
logger = logs.get_logger()
logger.debug("starting rsnap of ceph pool %s to qcows in %s/%s" %
(pool, settings.BACKUP_BASE_PATH, pool))
# get list of images from source
try:
names_on_source = ceph.gathernames(pool=pool)
except Exception as e:
logger.error('cannot get names from source with error %s' % e)
# fail out
raise NameError('cannot get names on source, failing run')
logger.info("names on source: %s" % ",".join(names_on_source))
# get list of images on backup dest already
try:
names_on_dest = get_names_on_dest(pool=pool)
except Exception as e:
logger.error('cannot get names from dest with error %s' % e)
# fail out
raise NameError('cannot get names on dest, failing run')
logger.info("names on dest: %s" % ",".join(names_on_dest))
# calculate difference
orphans_on_dest = [
image for image in names_on_dest if image not in names_on_source
]
if orphans_on_dest:
logger.info("orphans on dest: %s" % ",".join(orphans_on_dest))
# get template string for rsnap conf
template = templates.get_template()
successful = []
failed = []
orphans_rotated = []
orphans_failed_to_rotate = []
len_names = len(names_on_source)
index = 1
if len_names == 1 and names_on_source[0] == u'':
# TODO decide if this is critical/stop or just warn
logger.critical('no images found on source')
# sys.exit(1)
else:
for image in names_on_source:
# just to be safe, sanitize image names here too
try:
helpers.validate_string(image)
except NameError as e:
logger.error('bad character in image name %s: error %s' %
(image, e))
# fake return value from image
failed.append({
'image': image,
'pool': pool,
'successful': False,
'status': {
'export_qcow_ok': False,
'rsnap_ok': False,
'remove_qcow_ok': False
}
})
continue
# TODO catch other exceptions here?
logger.info('working on name %s of %s in pool %s: %s' %
(index, len_names, pool, image))
try:
result = rsnap_image(image, pool=pool, template=template)
if result['successful']:
logger.info('successfully done with %s' % image)
# store in array
successful.append(result)
else:
logger.error('error on %s : result: %s' %
(image, result['status']))
# store dict in dict
failed.append(result)
except Exception as e:
logger.error('error with pool %s at image %s' % (pool, image))
logger.exception(e)
# done with this image, increment counter
index = index + 1
# {'orphans_rotated': orphans_rotated, 'orphans_failed_to_rotate': orphans_failed_to_rotate}
if settings.NO_ROTATE_ORPHANS:
logger.info('not rotating orphans')
orphan_result = dict(
orphans_rotated=['no_rotate_orphans was set True'],
orphans_failed_to_rotate=['no_rotate_orphans was set True'])
else:
try:
orphan_result = rotate_orphans(orphans_on_dest, pool=pool)
except Exception as e:
logger.error('error with rotating orphans:')
logger.exception(e)
# just orphans so continue on
return ({
'successful':
successful,
'failed':
failed,
'orphans_rotated':
orphan_result['orphans_rotated'],
'orphans_failed_to_rotate':
orphan_result['orphans_failed_to_rotate'],
})
def rotate_orphans(orphans, pool=''):
logger = logs.get_logger()
if not pool:
pool = settings.POOL
backup_base_path = settings.BACKUP_BASE_PATH
# now check for ophans on dest
backup_path = "%s/%s" % (backup_base_path, pool)
orphans_rotated = []
orphans_failed_to_rotate = []
template = templates.get_template()
empty_tempdir = dirs.make_empty_tempdir()
for orphan in orphans:
logger.info('rotating orphan: %s' % orphan)
try:
# do this every time to be sure it's empty
dirs.check_empty_dir(empty_tempdir)
except NameError as e:
logger.error('error with verifying temp empty source,'
' cannot rotate orphans. error: %s' % e)
# fail out
return ({
'orphans_rotated':
orphans_rotated,
'orphans_failed_to_rotate': [
orphan for orphan in orphans
if orphan not in orphans_rotated
]
})
# note this uses temp_path on the dest - which we check to be empty
# note needs to end in a trailing /
source = "%s/" % empty_tempdir
conf_file = templates.write_conf(orphan,
pool=pool,
source=source,
template=template)
logger.info("rotating orphan %s" % orphan)
if settings.NOOP:
logger.info(
'NOOP: would have rotated orphan here using rsnapshot conf see previous lines'
)
else:
try:
rsnap_result = sh.rsnapshot(
'-c',
'%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, orphan),
settings.RETAIN_INTERVAL)
# if ssuccessful, log
if rsnap_result.stdout.strip("\n"):
logger.info("successful; stdout from rsnap:\n" +
rsnap_result.stdout.strip("\n"))
orphans_rotated.append({'pool': pool, 'orphan': orphan})
except sh.ErrorReturnCode as e:
orphans_failed_to_rotate.append({
'pool': pool,
'orphan': orphan
})
logger.error("failed to rotate orphan %s with code %s" %
(orphan, e.exit_code))
logger.error("stdout from source node:\n" +
e.stdout.strip("\n"))
logger.error("stderr from source node:\n" +
e.stderr.strip("\n"))
# unless flag to keep it for debug
if not settings.KEEPCONF:
templates.remove_conf(orphan, pool)
dirs.remove_empty_dir(empty_tempdir)
# TODO now check for any image dirs that are entirely empty and remove
# them (and the empty daily.NN inside them)
return ({
'orphans_rotated': orphans_rotated,
'orphans_failed_to_rotate': orphans_failed_to_rotate
})
def get_template():
logger = logs.get_logger()
env = jinja2.Environment(loader=jinja2.PackageLoader('ceph_rsnapshot'))
template = env.get_template('rsnapshot.template')
return template
def gathernames(pool='',
cephhost='',
cephuser='',
cephcluster='',
snap_naming_date_format='',
image_re='',
snap_date=''):
""" ssh to ceph node and get list of rbd images that match snap naming
format
"""
logger = logs.get_logger()
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not cephuser:
cephuser = settings.CEPH_USER
if not cephcluster:
cephcluster = settings.CEPH_CLUSTER
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
if not image_re:
image_re = settings.IMAGE_RE
RBD_LS_COMMAND = ('rbd ls %s --id=%s --cluster=%s --format=json' %
(pool, cephuser, cephcluster))
logger.info('sshing to %s and running %s to get list of images' %
(cephhost, RBD_LS_COMMAND))
try:
rbd_ls_result = sh.ssh(cephhost, RBD_LS_COMMAND)
except sh.ErrorReturnCode as e:
logger.info('error getting list of images from ceph node')
logger.exception(e.stderr)
raise
except Exception as e:
logger.info('error getting list of images from ceph node')
logger.exception(e)
raise
rbd_images_unfiltered = json.loads(rbd_ls_result.stdout)
logger.info('all images: %s' % ' '.join(rbd_images_unfiltered))
# filter by image_re
rbd_images_filtered = [
image for image in rbd_images_unfiltered if re.match(image_re, image)
]
logger.info('images after filtering by image_re /%s/ are: %s' %
(image_re, ' '.join(rbd_images_filtered)))
# first sanitize names of images
bad_names = []
for image in rbd_images_filtered:
try:
if helpers.validate_string(image):
continue
except NameError as e:
# bad character in a string, don't use this image
logger.warning('disallowed character in image name: %s'
' error %s' % (image, e))
# take it out of the good array
rbd_images_filtered.remove(image)
bad_names.append(image)
except Exception as e:
raise
# now check for snaps
images_with_snaps = []
images_without_snaps = []
for image in rbd_images_filtered:
if check_snap(image):
images_with_snaps.append(image)
else:
images_without_snaps.append(image)
logger.info('images with snaps are: %s' % ' '.join(images_with_snaps))
if images_without_snaps:
logger.warning('note, found %s images with no snaps' %
len(images_without_snaps))
if bad_names:
logger.warning('note, found %s images with bad names' % len(bad_names))
return images_with_snaps
def export_qcow(image,
snap='',
pool='',
cephhost='',
cephuser='',
cephcluster='',
noop=None,
snap_naming_date_format='',
snap_date=''):
"""ssh to ceph node, check free space vs rbd provisioned size,
and export a qcow to qcow_temp_path/pool/imagename.qcow2
"""
logger = logs.get_logger()
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
if not snap:
snap = get_snapdate(snap_naming_date_format=snap_naming_date_format,
snap_date=snap_date)
if not pool:
pool = settings.POOL
if not cephhost:
cephhost = settings.CEPH_HOST
if not cephuser:
cephuser = settings.CEPH_USER
if not cephcluster:
cephcluster = settings.CEPH_CLUSTER
if not noop:
noop = settings.NOOP
logger.info(
'going to export image %s@%s from pool %s on ceph host %s cluster %s'
' as user %s' % (image, snap, pool, cephhost, cephcluster, cephuser))
# if any of these errors, fail this export and raise the errors up
avail_bytes = get_freespace(settings.QCOW_TEMP_PATH)
rbd_image_used_size = get_rbd_size(image)
logger.info("image size %s" % rbd_image_used_size)
if rbd_image_used_size > (avail_bytes - settings.MIN_FREESPACE):
logger.error("not enough free space to export this qcow")
raise NameError('not enough space to export this qcow')
# build source and dest strings
qemu_source_string = ("rbd:%s/%s@%s:id=%s:conf=/etc/ceph/%s.conf" %
(pool, image, snap, cephuser, cephcluster))
qemu_dest_string = "%s/%s/%s@%s.qcow2" % (settings.QCOW_TEMP_PATH, pool,
image, snap)
# do the export
QEMU_IMG_COMMAND = ('qemu-img convert %s %s -f raw'
' -O qcow2' % (qemu_source_string, qemu_dest_string))
logger.info('running rbd export on ceph host %s with command %s' %
(cephhost, QEMU_IMG_COMMAND))
try:
ts = time.time()
if noop:
logger.info('NOOP: would have exported qcow')
else:
export_result = sh.ssh(cephhost, QEMU_IMG_COMMAND)
tf = time.time()
elapsed_time = tf - ts
elapsed_time_ms = elapsed_time * 10**3
except sh.ErrorReturnCode as e:
logger.error('error exporting qcow with command %s on ceph host %s,'
' output from ssh:' % (cephhost, QEMU_IMG_COMMAND))
logger.exception(e.stderr)
raise
except Exception as e:
logger.error('error exporting qcow with command %s on ceph host %s,'
' output from ssh:' % (cephhost, QEMU_IMG_COMMAND))
logger.exception(e)
raise
logger.info('qcow exported in %s ms' % elapsed_time_ms)
return elapsed_time_ms
def write_conf(image,
pool='',
source='',
template='',
snap_naming_date_format='',
snap_date='',
snap=''):
if not snap_naming_date_format:
snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT
if not snap_date:
snap_date = settings.SNAP_DATE
if not snap:
snap = ceph.get_snapdate(
snap_naming_date_format=snap_naming_date_format,
snap_date=snap_date)
if not pool:
pool = settings.POOL
host = settings.CEPH_HOST
temp_path = settings.QCOW_TEMP_PATH
backup_base_path = settings.BACKUP_BASE_PATH
# get logger we setup earlier
logger = logs.get_logger()
# only reopen template if we don't have it - ie, are we part of a pool run
if not template:
template = get_template()
# create source path string if an override wasn't passed to us
# note using the . to tell rsnap where to start the relative dir
if source == '':
source = 'root@%s:%s/%s/.' % (settings.CEPH_HOST,
settings.QCOW_TEMP_PATH, pool)
destination = '%s/%s/%s' % (settings.BACKUP_BASE_PATH, settings.POOL,
image)
logger.info('writing conf for image %s to rsnap from %s to %s' %
(image, source, destination))
my_template = template.render(nickname=image,
pool=pool,
source=source,
destination=destination,
retain_interval=settings.RETAIN_INTERVAL,
retain_number=settings.RETAIN_NUMBER,
log_base_path=settings.LOG_BASE_PATH,
subdir='.',
extra_args=settings.EXTRA_ARGS)
if settings.NOOP:
logger.info('NOOP: would have written conf file to %s/%s/%s.conf' %
(settings.TEMP_CONF_DIR, pool, image))
logger.info('NOOP: conf file contents would have been: \n%s' %
my_template)
# fake conf file name to return
return '%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image)
else:
# conf file of the form /tmp_conf_dir/pool/imagename.conf
try:
conf_file = open(
'%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image), 'w')
conf_file.write(my_template)
except Exception as e:
logger.error('error with temp conf file for orphan: %s error: %s' %
(image, e))
raise
return conf_file.name
