def setup_temp_conf_dir_for_pool(pool=''): """ setup temp conf dir for rsnap confs if provided, use, otherwise make temp dir """ if not pool: pool = settings.POOL logger = logs.get_logger() if settings.TEMP_CONF_DIR: if os.path.isdir(settings.TEMP_CONF_DIR): logger.info('using temp conf dir %s' % settings.TEMP_CONF_DIR) else: try: setup_dir(settings.TEMP_CONF_DIR) except IOError as e: logger.error( 'Cannot create conf temp dir (or intermediate dirs) from' + ' setting %s with error %s' % (settings.TEMP_CONF_DIR, e)) raise else: try: logger.info('making a tempdir for rsnap confs') temp_conf_dir = make_empty_tempdir( prefix=settings.TEMP_CONF_DIR_PREFIX) # store this in global settings settings.TEMP_CONF_DIR = temp_conf_dir except IOError as e: logger.error('cannot create conf temp dir with error %s' % e) raise # now make for pool pool_conf_dir = "%s/%s" % (settings.TEMP_CONF_DIR, pool) setup_dir(pool_conf_dir) return settings.TEMP_CONF_DIR
def remove_snap_status_file(snap_date, cephhost='', snap_status_file_path='', noop=''): logger = logs.get_logger() if not cephhost: cephhost = settings.CEPH_HOST if not snap_status_file_path: snap_status_file_path = settings.SNAP_STATUS_FILE_PATH if not noop: noop = settings.NOOP REMOVE_SNAP_STATUS_FILE_COMMAND = ('rm -fv %s/%s' % (snap_status_file_path, snap_date)) logger.info('removing snap status file on ceph host with command %s' % REMOVE_SNAP_STATUS_FILE_COMMAND) if noop: logger.info('would have run %s' % REMOVE_SNAP_STATUS_FILE_COMMAND) remove_snap_status_file_result = 'noop' else: remove_snap_status_file_result = sh.ssh( cephhost, REMOVE_SNAP_STATUS_FILE_COMMAND).strip('\n') # TODO handle some errors gracefully here logger.info("done removing snap status file: %s" % remove_snap_status_file_result) return True
def remove_empty_dir(directory): """ remove a directory if it's empty """ logger = logs.get_logger() if settings.NOOP: logger.info('NOOP: would have removed %s' % directory) else: logger.info('removing %s' % directory) os.rmdir(directory)
def remove_qcow(image, pool='', cephhost='', cephuser='', cephcluster='', snap_naming_date_format='', snap_date='', snap='', noop=None): """ ssh to ceph node and remove a qcow from path qcow_temp_path/pool/imagename.qcow2 """ logger = logs.get_logger() if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE if not snap: snap = get_snapdate(snap_naming_date_format=snap_naming_date_format, snap_date=snap_date) if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not cephuser: cephuser = settings.CEPH_USER # TODO use ceph cluster in path naming if not cephcluster: cephcluster = settings.CEPH_CLUSTER if not noop: noop = settings.NOOP temp_qcow_file = ("%s/%s/%s@%s.qcow2" % (settings.QCOW_TEMP_PATH, settings.POOL, image, snap)) logger.info("deleting temp qcow from path %s on ceph host %s" % (temp_qcow_file, cephhost)) SSH_RM_QCOW_COMMAND = 'rm %s' % temp_qcow_file try: if settings.NOOP: logger.info('NOOP: would have removed temp qcow for image %s from' ' ceph host %s with command %s' % (image, cephhost, SSH_RM_QCOW_COMMAND)) else: sh.ssh(cephhost, SSH_RM_QCOW_COMMAND) except sh.ErrorReturnCode as e: logger.error('error removing temp qcow %s with error from ssh:' % temp_qcow_file) logger.exception(e.stderr) raise except Exception as e: logger.error('error removing temp qcow %s' % temp_qcow_file) logger.exception(e) raise logger.info("successfully removed qcow for %s" % image) return True
def remove_conf(image, pool=''): if not pool: pool = settings.POOL # get logger we setup earlier logger = logs.get_logger() if settings.NOOP: logger.info('NOOP: would have removed conf file %s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image)) else: logger.info('removing temp rsnap conf file for image %s' % image) os.remove('%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image))
def setup_log_dirs_for_pool(pool=''): """ wrapper of the above to setup log dirs """ logger = logs.get_logger() if not pool: pool = settings.POOL dirs = [ settings.LOG_BASE_PATH, "%s/rsnap" % settings.LOG_BASE_PATH, "%s/rsnap/%s" % (settings.LOG_BASE_PATH, pool), ] for directory in dirs: setup_dir(directory, perms=0o755)
def make_empty_tempdir(prefix=''): """ make an empty tempdir """ logger = logs.get_logger() if not prefix: prefix = 'empty_' if settings.NOOP: logger.info('NOOP: would have made a tempdir with prefix %s' % prefix) return 'noop_fake_empty_path_with_prefix_%s' % prefix else: logger.info('creating tempdir with prefix %s' % prefix) empty_tempdir = tempfile.mkdtemp(prefix=prefix) return empty_tempdir
def setup_dir(directory, perms=0o700): logger = logs.get_logger() if not os.path.isdir(directory): # make dir and preceeding dirs if necessary if settings.NOOP: logger.info('NOOP: would have run makedirs on path %s' % directory) else: logger.info('creating directory %o %s' % (perms, directory)) os.makedirs(directory, perms) else: logger.info('directory %s already exists, so using it' % directory) # still need to check perms check_set_dir_perms(directory, perms)
def check_snap_status_file(cephhost='', snap_status_file_path=''): logger = logs.get_logger() if not cephhost: cephhost = settings.CEPH_HOST if not snap_status_file_path: snap_status_file_path = settings.SNAP_STATUS_FILE_PATH CHECK_SNAP_STATUS_DIR_COMMAND = ('ls -t %s/*' % snap_status_file_path) logger.info('checking snap status directory %s on ceph host' % snap_status_file_path) try: snap_status_dir_result = sh.ssh( cephhost, CHECK_SNAP_STATUS_DIR_COMMAND).strip('\n') except sh.ErrorReturnCode as e: if e.exit_code == 2: raise exceptions.NoSnapStatusFilesFoundError( cephhost=cephhost, status_dir=snap_status_file_path, e=e) else: raise logger.debug("found: %s" % snap_status_dir_result) snap_dates = [ snap_status_file.split('/')[-1] for snap_status_file in snap_status_dir_result.split('\n') ] logger.debug("snap dates:") logger.debug(snap_dates) snap_date = snap_dates[0] logger.debug("checking newest snap_date %s" % snap_date) try: result = check_formatted_snap_date(snap_date=snap_date) except exceptions.SnapDateNotValidDateError as e: raise # if we're here it was a valid date and matches format # remove the rest of them that match for old_snap_date in snap_dates[1:]: logger.warn( "found old snap_date files- checking and removing if valid dates") try: logger.debug('checking old snap date %s' % old_snap_date) check_formatted_snap_date(snap_date=old_snap_date) except exceptions.SnapDateNotValidDateError as e: e.log(warn=True) continue except exceptions.SnapDateFormatMismatchError as e: e.log(warn=True) continue # if here then it's a valid date logger.warning('removing old snap_date status file %s because we have' ' a newer one %s' % (old_snap_date, snap_date)) remove_snap_status_file(snap_date=old_snap_date) logger.info('using snap_date %s found from queue on ceph host' % snap_date) return snap_date
def get_rbd_size(image, snap='', pool='', cephhost='', cephuser='', cephcluster='', snap_naming_date_format='', snap_date=''): """ssh to ceph node check the size of this image@snap """ logger = logs.get_logger() if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE if not snap: snap = get_snapdate(snap_naming_date_format=snap_naming_date_format, snap_date=snap_date) if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not cephuser: cephuser = settings.CEPH_USER if not cephcluster: cephcluster = settings.CEPH_CLUSTER rbd_image_string = "%s/%s@%s" % (pool, image, snap) RBD_COMMAND = ('rbd du %s --user=%s --cluster=%s --format=json' % (rbd_image_string, cephuser, cephcluster)) logger.info('getting rbd size from ceph host %s with command %s' % (cephhost, RBD_COMMAND)) try: rbd_du_result = sh.ssh(cephhost, RBD_COMMAND) rbd_image_used_size = (json.loads( rbd_du_result.stdout)['images'][0]['used_size']) rbd_image_provisioned_size = (json.loads( rbd_du_result.stdout)['images'][0]['provisioned_size']) # using provisioned_size as these are snaps and space could be on the # parent return rbd_image_provisioned_size except sh.ErrorReturnCode as e: logger.error('error getting rbd size for %s, output from ssh:' % rbd_image_string) logger.exception(e.stderr) raise except Exception as e: logger.error('error getting rbd size for %s' % rbd_image_string) logger.exception(e) raise
def check_set_dir_perms(directory, perms=0o700): logger = logs.get_logger() desired_mode = oct(perms)[-3:] if settings.NOOP: logger.info('NOOP: would have verified that permissions on %s are %s' % (directory, desired_mode)) else: dir_stat = os.stat(directory) current_mode = oct(dir_stat.st_mode)[-3:] if current_mode != desired_mode: logger.warning( 'perms not correct on %s: currently %s should be %s,' 'fixing' % (directory, current_mode, desired_mode)) os.chmod(directory, perms) logger.info('perms now correctly set to %s on %s' % (desired_mode, directory))
def setup_backup_dirs_for_pool(pool='', dirs=''): """ wrapper of the above to setup backup dirs """ logger = logs.get_logger() if not pool: pool = settings.POOL if not dirs: dirs = [ settings.BACKUP_BASE_PATH, "%s/%s" % (settings.BACKUP_BASE_PATH, pool), ] # TODO support multiple pools # for pool in settings.POOLS # dirs.append(pool) for directory in dirs: logger.info('setting up backup dir: %s' % directory) setup_dir(directory)
def remove_temp_conf_dir(): """ remove temp conf dirs """ logger = logs.get_logger() if not settings.KEEPCONF: logger.info("removing temp conf dir %s" % settings.TEMP_CONF_DIR) try: if settings.NOOP: logger.info('NOOP: would have removed temp conf dirs %s/%s and' ' %s' % (settings.TEMP_CONF_DIR, settings.POOL, settings.TEMP_CONF_DIR)) else: # TODO all pools os.rmdir("%s/%s" % (settings.TEMP_CONF_DIR, settings.POOL)) os.rmdir(settings.TEMP_CONF_DIR) except (IOError, OSError) as e: logger.warning("unable to remove temp conf dir with error %s" % e)
def get_snapdate(snap_naming_date_format='', snap_date=''): """get todays date in iso format, this can run on either node """ logger = logs.get_logger() if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE try: converted_snap_date = sh.date('+%s' % snap_naming_date_format, date=snap_date).strip('\n') except sh.ErrorReturnCode as e: if e.exit_code == 1: raise (exceptions.SnapDateNotValidDateError( snap_date=snap_date, date_format=snap_naming_date_format, e=e)) else: raise return converted_snap_date
def validate_settings_strings(): """ check all settings strings to make sure they are only safe chars if not fail run """ logger = logs.get_logger() logger.debug('checking settings strings to ensure they only contain safe' ' chars: %s' % settings.STRING_SAFE_CHAR_RE) # check strings are safe current_settings = get_current_settings() for key in current_settings: if key in settings.ADDITIONAL_SAFE_CHARS: additional_safe_chars = settings.ADDITIONAL_SAFE_CHARS[key] else: additional_safe_chars = '' if key in ['IMAGE_RE']: # these are allowed to have weird characters # also this is only used in this script as a RE continue value = current_settings[key] if type(value) in [bool, int]: # don't compare these to an RE continue try: if key == 'POOLS': logger.debug('checking pools string %s' % value) pools_arr = value.split(',') for pool in pools_arr: logger.debug('checking string for pool %s' % pool) validate_string( pool, additional_safe_chars=additional_safe_chars) # if here then they all validated continue if validate_string(value, additional_safe_chars=additional_safe_chars): continue except NameError as e: # bad character in a string, fail run raise NameError('disallowed character in setting: %s' ' error %s' % (key, e)) except Exception as e: raise logger.debug('all settings strings ok')
def check_snap(image, snap='', pool='', cephhost='', cephuser='', cephcluster='', snap_naming_date_format='', snap_date=''): """ ssh to ceph host and check for a snapshot """ logger = logs.get_logger() if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE if not snap: snap = get_snapdate(snap_naming_date_format=snap_naming_date_format, snap_date=snap_date) if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not cephuser: cephuser = settings.CEPH_USER if not cephcluster: cephcluster = settings.CEPH_CLUSTER RBD_CHECK_SNAP_COMMAND = ('rbd info %s/%s@%s --id=%s --cluster=%s' % (pool, image, snap, cephuser, cephcluster)) logger.info('checking for snap with command %s' % RBD_CHECK_SNAP_COMMAND) try: rbd_check_result = sh.ssh(cephhost, RBD_CHECK_SNAP_COMMAND) except sh.ErrorReturnCode as e: # this just means no snap found, log but don't raise logger.warning('no snap found for image %s' % image) # return false we didn't find one return False except Exception as e: logger.info('error getting list of images from ceph node') logger.exception(e) raise # if here then rbd snap found, so return True return True
def get_names_on_dest(pool=''): logger = logs.get_logger() if not pool: pool = settings.POOL backup_base_path = settings.BACKUP_BASE_PATH # get logger we setup earlier logger = logging.getLogger('ceph_rsnapshot') backup_path = "%s/%s" % (backup_base_path, pool) try: names_on_dest = os.listdir(backup_path) except (IOError, OSError) as e: if settings.NOOP: # this will fail if noop and the dir doesn't exist, so # fake nothing there and move on logger.info('NOOP: would have listed vms in directory %s' % backup_path) return [] logger.error(e) raise NameError('get_names_on_dest failed with error %s' % e) # FIXME cehck error return names_on_dest
def get_freespace(path=''): """ssh to ceph node and get freespace for a path if not specified, use settings.QCOW_TEMP_PATH/settings.POOL """ logger = logs.get_logger() if not path: path = "%s/%s" % (settings.QCOW_TEMP_PATH, settings.POOL) DF_COMMAND = ("LANG='' LC_CTYPE='' df -P %s | grep / | awk '{print $4}';" " ( exit ${PIPESTATUS[0]} )" % path) try: free_space_kb = sh.ssh(settings.CEPH_HOST, DF_COMMAND) free_space_bytes = int(free_space_kb.stdout) * 1024 return free_space_bytes except sh.ErrorReturnCode as e: logger.error('error getting free space on ceph node for %s' % path) logger.exception(e.stderr) # pass it up raise except Exception as e: logger.error('error getting free space on ceph node') logger.exception(e) # pass it up raise
def check_qcow_temp_path_empty_for_pool(cephhost='', qcowtemppath='', pool='', noop=None): logger = logs.get_logger() if not cephhost: cephhost = settings.CEPH_HOST if not qcowtemppath: qcowtemppath = settings.QCOW_TEMP_PATH if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not noop: noop = settings.NOOP if noop: # this dir might not exist yet so just say it's good and move on return True temp_path = '%s/%s' % (qcowtemppath, pool) logger.info('checking qcow temp export path %s is empty on ceph host' ' %s' % (temp_path, cephhost)) LS_COMMAND = 'ls -a %s' % temp_path try: ls_result = sh.ssh(cephhost, LS_COMMAND) if ls_result == EMPTY_DIR_LS_RESULT: return True else: logger.error('ERROR: temp qcow export directory %s not empty: %s', temp_path, ls_result) except sh.ErrorReturnCode as e: logger.error('error checking temp qcow export directory') logger.exception(e.stderr) raise except Exception as e: logger.error('error checking temp qcow export directory') logger.exception(e) raise
def rsnap_image_sh(image, pool=''): logger = logs.get_logger() if not pool: pool = settings.POOL # TODO check free space before rsnapping logger.info("rsnapping %s" % image) rsnap_conf_file = '%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image) if settings.NOOP: logger.info( 'NOOP: would have rsnapshotted image from conf file ' '%s/%s/%s.conf for retain interval %s ' % (settings.TEMP_CONF_DIR, pool, image, settings.RETAIN_INTERVAL)) # set this False so it's clear this wasn't successful as it was a noop rsnap_ok = False else: try: ts = time.time() rsnap_result = sh.rsnapshot('-c', rsnap_conf_file, settings.RETAIN_INTERVAL) tf = time.time() elapsed_time = tf - ts elapsed_time_ms = elapsed_time * 10**3 rsnap_ok = True logger.info("rsnap successful for image %s in %sms" % (image, elapsed_time_ms)) if rsnap_result.stdout.strip("\n"): logger.info("stdout from rsnap:\n" + rsnap_result.stdout.strip("\n")) except sh.ErrorReturnCode as e: logger.error("failed to rsnap %s with code %s" % (image, e.exit_code)) # TODO move log formatting and writing to a function logger.error("stdout from rsnap:\n" + e.stdout.strip("\n")) logger.error("stderr from rsnap:\n" + e.stderr.strip("\n")) rsnap_ok = False return rsnap_ok
def rsnap_image(image, pool='', template=None): if not pool: pool = settings.POOL temp_path = settings.QCOW_TEMP_PATH extra_args = settings.EXTRA_ARGS conf_base_path = settings.TEMP_CONF_DIR backup_base_path = settings.BACKUP_BASE_PATH # get logger we setup earlier logger = logs.get_logger() logger.info('working on image %s in pool %s' % (image, pool)) # setup flags qcow_temp_path_empty = False export_qcow_ok = False rsnap_ok = False remove_qcow_ok = False # only reopen if we haven't pulled this yet - ie, are we part of a pool run if not template: template = templates.get_template() # create the temp conf file conf_file = templates.write_conf(image, pool=pool, template=template) logger.info(conf_file) # make sure temp qcow dir is empty try: if dirs.check_qcow_temp_path_empty_for_pool(pool=pool): qcow_temp_path_empty = True except Exception as e: # if it's not empty, fail this image logger.error('qcow temp path not empty, failing this image') logger.exception(e) qcow_temp_path_empty = False # ssh to source and export temp qcow of this image if qcow_temp_path_empty: try: ceph.export_qcow(image, pool=pool) export_qcow_ok = True except NameError as e: # probably not enough space. set to false and try to go and remove this # one, or go to next image in case it was temporary logger.error('error from export qcow: %s' % e) export_qcow_ok = False except Exception as e: logger.error('error from export qcow') logger.exception(e) export_qcow_ok = False # if exported ok, then rsnap this image if export_qcow_ok: try: rsnap_ok = rsnap_image_sh(image, pool=pool) except Exception as e: # TODO logger.error('error with rsnapping image %s' % image) else: logger.error( "skipping rsnap of image %s because export to qcow failed" % image) # either way remove the temp qcow logger.info("removing temp qcow for %s" % image) try: remove_qcow_ok = ceph.remove_qcow(image, pool=pool) except Exception as e: logger.error( 'error removing qcow. will continue to next image anyways,' ' note that we check for free space so wont entirely fill disk if they' ' all fail') # either way remove the temp conf file # unless flag to keep it for debug if not settings.KEEPCONF: templates.remove_conf(image, pool=pool) if export_qcow_ok and rsnap_ok and remove_qcow_ok: successful = True else: successful = False # return a blob with the details return ({ 'image': image, 'pool': pool, 'successful': successful, 'status': { 'qcow_temp_path_empty': qcow_temp_path_empty, 'export_qcow_ok': export_qcow_ok, 'rsnap_ok': rsnap_ok, 'remove_qcow_ok': remove_qcow_ok } })
def setup_qcow_temp_path(pool='', cephhost='', qcowtemppath='', noop=None): """ ssh to ceph node and check or make temp qcow export path """ logger = logs.get_logger() if not qcowtemppath: qcowtemppath = settings.QCOW_TEMP_PATH if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not noop: noop = settings.NOOP temp_path = '%s/%s' % (qcowtemppath, pool) logger.info('making qcow temp export path %s on ceph host %s' % (temp_path, cephhost)) LS_COMMAND = 'ls %s' % temp_path MKDIR_COMMAND = 'mkdir -p %s' % temp_path CHMOD_COMMAND = 'LANG=' ' LC_CTYPE=' ' chmod 700 %s' % temp_path try: ls_result = sh.ssh(cephhost, LS_COMMAND) except sh.ErrorReturnCode as e: if e.exit_code == 2: # ls returns 2 for no such dir, this is OK, just make it try: if noop: logger.info('NOOP: would have made qcow temp path %s' % temp_path) else: sh.ssh(cephhost, MKDIR_COMMAND) sh.ssh(cephhost, CHMOD_COMMAND) except sh.ErrorReturnCode as e: logger.error('error making or chmodding qcow temp dir:') logger.exception(e.stderr) raise except Exception as e: logger.error('error making or chmodding qcow temp dir') logger.exception(e) raise else: logger.error('error checking temp qcow export directory') logger.exception(e.stderr) raise except Exception as e: logger.error('error checking temp qcow export directory') logger.exception(e) raise logger.info('using qcow temp path: %s' % temp_path) # now just to be safe verify perms on it are 700 try: if settings.NOOP: logger.info( 'NOOP: would have chmodded qcow temp path with command:' ' %s' % CHMOD_COMMAND) else: sh.ssh(cephhost, CHMOD_COMMAND) except sh.ErrorReturnCode as e: logger.error('error chmodding qcow temp dir:') logger.exception(e.stderr) raise except Exception as e: logger.error('error chmodding qcow temp dir') logger.exception(e) raise
def rsnap_pool(pool): # get values from settings host = settings.CEPH_HOST # get logger we setup earlier logger = logs.get_logger() logger.debug("starting rsnap of ceph pool %s to qcows in %s/%s" % (pool, settings.BACKUP_BASE_PATH, pool)) # get list of images from source try: names_on_source = ceph.gathernames(pool=pool) except Exception as e: logger.error('cannot get names from source with error %s' % e) # fail out raise NameError('cannot get names on source, failing run') logger.info("names on source: %s" % ",".join(names_on_source)) # get list of images on backup dest already try: names_on_dest = get_names_on_dest(pool=pool) except Exception as e: logger.error('cannot get names from dest with error %s' % e) # fail out raise NameError('cannot get names on dest, failing run') logger.info("names on dest: %s" % ",".join(names_on_dest)) # calculate difference orphans_on_dest = [ image for image in names_on_dest if image not in names_on_source ] if orphans_on_dest: logger.info("orphans on dest: %s" % ",".join(orphans_on_dest)) # get template string for rsnap conf template = templates.get_template() successful = [] failed = [] orphans_rotated = [] orphans_failed_to_rotate = [] len_names = len(names_on_source) index = 1 if len_names == 1 and names_on_source[0] == u'': # TODO decide if this is critical/stop or just warn logger.critical('no images found on source') # sys.exit(1) else: for image in names_on_source: # just to be safe, sanitize image names here too try: helpers.validate_string(image) except NameError as e: logger.error('bad character in image name %s: error %s' % (image, e)) # fake return value from image failed.append({ 'image': image, 'pool': pool, 'successful': False, 'status': { 'export_qcow_ok': False, 'rsnap_ok': False, 'remove_qcow_ok': False } }) continue # TODO catch other exceptions here? logger.info('working on name %s of %s in pool %s: %s' % (index, len_names, pool, image)) try: result = rsnap_image(image, pool=pool, template=template) if result['successful']: logger.info('successfully done with %s' % image) # store in array successful.append(result) else: logger.error('error on %s : result: %s' % (image, result['status'])) # store dict in dict failed.append(result) except Exception as e: logger.error('error with pool %s at image %s' % (pool, image)) logger.exception(e) # done with this image, increment counter index = index + 1 # {'orphans_rotated': orphans_rotated, 'orphans_failed_to_rotate': orphans_failed_to_rotate} if settings.NO_ROTATE_ORPHANS: logger.info('not rotating orphans') orphan_result = dict( orphans_rotated=['no_rotate_orphans was set True'], orphans_failed_to_rotate=['no_rotate_orphans was set True']) else: try: orphan_result = rotate_orphans(orphans_on_dest, pool=pool) except Exception as e: logger.error('error with rotating orphans:') logger.exception(e) # just orphans so continue on return ({ 'successful': successful, 'failed': failed, 'orphans_rotated': orphan_result['orphans_rotated'], 'orphans_failed_to_rotate': orphan_result['orphans_failed_to_rotate'], })
def rotate_orphans(orphans, pool=''): logger = logs.get_logger() if not pool: pool = settings.POOL backup_base_path = settings.BACKUP_BASE_PATH # now check for ophans on dest backup_path = "%s/%s" % (backup_base_path, pool) orphans_rotated = [] orphans_failed_to_rotate = [] template = templates.get_template() empty_tempdir = dirs.make_empty_tempdir() for orphan in orphans: logger.info('rotating orphan: %s' % orphan) try: # do this every time to be sure it's empty dirs.check_empty_dir(empty_tempdir) except NameError as e: logger.error('error with verifying temp empty source,' ' cannot rotate orphans. error: %s' % e) # fail out return ({ 'orphans_rotated': orphans_rotated, 'orphans_failed_to_rotate': [ orphan for orphan in orphans if orphan not in orphans_rotated ] }) # note this uses temp_path on the dest - which we check to be empty # note needs to end in a trailing / source = "%s/" % empty_tempdir conf_file = templates.write_conf(orphan, pool=pool, source=source, template=template) logger.info("rotating orphan %s" % orphan) if settings.NOOP: logger.info( 'NOOP: would have rotated orphan here using rsnapshot conf see previous lines' ) else: try: rsnap_result = sh.rsnapshot( '-c', '%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, orphan), settings.RETAIN_INTERVAL) # if ssuccessful, log if rsnap_result.stdout.strip("\n"): logger.info("successful; stdout from rsnap:\n" + rsnap_result.stdout.strip("\n")) orphans_rotated.append({'pool': pool, 'orphan': orphan}) except sh.ErrorReturnCode as e: orphans_failed_to_rotate.append({ 'pool': pool, 'orphan': orphan }) logger.error("failed to rotate orphan %s with code %s" % (orphan, e.exit_code)) logger.error("stdout from source node:\n" + e.stdout.strip("\n")) logger.error("stderr from source node:\n" + e.stderr.strip("\n")) # unless flag to keep it for debug if not settings.KEEPCONF: templates.remove_conf(orphan, pool) dirs.remove_empty_dir(empty_tempdir) # TODO now check for any image dirs that are entirely empty and remove # them (and the empty daily.NN inside them) return ({ 'orphans_rotated': orphans_rotated, 'orphans_failed_to_rotate': orphans_failed_to_rotate })
def get_template(): logger = logs.get_logger() env = jinja2.Environment(loader=jinja2.PackageLoader('ceph_rsnapshot')) template = env.get_template('rsnapshot.template') return template
def gathernames(pool='', cephhost='', cephuser='', cephcluster='', snap_naming_date_format='', image_re='', snap_date=''): """ ssh to ceph node and get list of rbd images that match snap naming format """ logger = logs.get_logger() if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not cephuser: cephuser = settings.CEPH_USER if not cephcluster: cephcluster = settings.CEPH_CLUSTER if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE if not image_re: image_re = settings.IMAGE_RE RBD_LS_COMMAND = ('rbd ls %s --id=%s --cluster=%s --format=json' % (pool, cephuser, cephcluster)) logger.info('sshing to %s and running %s to get list of images' % (cephhost, RBD_LS_COMMAND)) try: rbd_ls_result = sh.ssh(cephhost, RBD_LS_COMMAND) except sh.ErrorReturnCode as e: logger.info('error getting list of images from ceph node') logger.exception(e.stderr) raise except Exception as e: logger.info('error getting list of images from ceph node') logger.exception(e) raise rbd_images_unfiltered = json.loads(rbd_ls_result.stdout) logger.info('all images: %s' % ' '.join(rbd_images_unfiltered)) # filter by image_re rbd_images_filtered = [ image for image in rbd_images_unfiltered if re.match(image_re, image) ] logger.info('images after filtering by image_re /%s/ are: %s' % (image_re, ' '.join(rbd_images_filtered))) # first sanitize names of images bad_names = [] for image in rbd_images_filtered: try: if helpers.validate_string(image): continue except NameError as e: # bad character in a string, don't use this image logger.warning('disallowed character in image name: %s' ' error %s' % (image, e)) # take it out of the good array rbd_images_filtered.remove(image) bad_names.append(image) except Exception as e: raise # now check for snaps images_with_snaps = [] images_without_snaps = [] for image in rbd_images_filtered: if check_snap(image): images_with_snaps.append(image) else: images_without_snaps.append(image) logger.info('images with snaps are: %s' % ' '.join(images_with_snaps)) if images_without_snaps: logger.warning('note, found %s images with no snaps' % len(images_without_snaps)) if bad_names: logger.warning('note, found %s images with bad names' % len(bad_names)) return images_with_snaps
def export_qcow(image, snap='', pool='', cephhost='', cephuser='', cephcluster='', noop=None, snap_naming_date_format='', snap_date=''): """ssh to ceph node, check free space vs rbd provisioned size, and export a qcow to qcow_temp_path/pool/imagename.qcow2 """ logger = logs.get_logger() if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE if not snap: snap = get_snapdate(snap_naming_date_format=snap_naming_date_format, snap_date=snap_date) if not pool: pool = settings.POOL if not cephhost: cephhost = settings.CEPH_HOST if not cephuser: cephuser = settings.CEPH_USER if not cephcluster: cephcluster = settings.CEPH_CLUSTER if not noop: noop = settings.NOOP logger.info( 'going to export image %s@%s from pool %s on ceph host %s cluster %s' ' as user %s' % (image, snap, pool, cephhost, cephcluster, cephuser)) # if any of these errors, fail this export and raise the errors up avail_bytes = get_freespace(settings.QCOW_TEMP_PATH) rbd_image_used_size = get_rbd_size(image) logger.info("image size %s" % rbd_image_used_size) if rbd_image_used_size > (avail_bytes - settings.MIN_FREESPACE): logger.error("not enough free space to export this qcow") raise NameError('not enough space to export this qcow') # build source and dest strings qemu_source_string = ("rbd:%s/%s@%s:id=%s:conf=/etc/ceph/%s.conf" % (pool, image, snap, cephuser, cephcluster)) qemu_dest_string = "%s/%s/%s@%s.qcow2" % (settings.QCOW_TEMP_PATH, pool, image, snap) # do the export QEMU_IMG_COMMAND = ('qemu-img convert %s %s -f raw' ' -O qcow2' % (qemu_source_string, qemu_dest_string)) logger.info('running rbd export on ceph host %s with command %s' % (cephhost, QEMU_IMG_COMMAND)) try: ts = time.time() if noop: logger.info('NOOP: would have exported qcow') else: export_result = sh.ssh(cephhost, QEMU_IMG_COMMAND) tf = time.time() elapsed_time = tf - ts elapsed_time_ms = elapsed_time * 10**3 except sh.ErrorReturnCode as e: logger.error('error exporting qcow with command %s on ceph host %s,' ' output from ssh:' % (cephhost, QEMU_IMG_COMMAND)) logger.exception(e.stderr) raise except Exception as e: logger.error('error exporting qcow with command %s on ceph host %s,' ' output from ssh:' % (cephhost, QEMU_IMG_COMMAND)) logger.exception(e) raise logger.info('qcow exported in %s ms' % elapsed_time_ms) return elapsed_time_ms
def write_conf(image, pool='', source='', template='', snap_naming_date_format='', snap_date='', snap=''): if not snap_naming_date_format: snap_naming_date_format = settings.SNAP_NAMING_DATE_FORMAT if not snap_date: snap_date = settings.SNAP_DATE if not snap: snap = ceph.get_snapdate( snap_naming_date_format=snap_naming_date_format, snap_date=snap_date) if not pool: pool = settings.POOL host = settings.CEPH_HOST temp_path = settings.QCOW_TEMP_PATH backup_base_path = settings.BACKUP_BASE_PATH # get logger we setup earlier logger = logs.get_logger() # only reopen template if we don't have it - ie, are we part of a pool run if not template: template = get_template() # create source path string if an override wasn't passed to us # note using the . to tell rsnap where to start the relative dir if source == '': source = 'root@%s:%s/%s/.' % (settings.CEPH_HOST, settings.QCOW_TEMP_PATH, pool) destination = '%s/%s/%s' % (settings.BACKUP_BASE_PATH, settings.POOL, image) logger.info('writing conf for image %s to rsnap from %s to %s' % (image, source, destination)) my_template = template.render(nickname=image, pool=pool, source=source, destination=destination, retain_interval=settings.RETAIN_INTERVAL, retain_number=settings.RETAIN_NUMBER, log_base_path=settings.LOG_BASE_PATH, subdir='.', extra_args=settings.EXTRA_ARGS) if settings.NOOP: logger.info('NOOP: would have written conf file to %s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image)) logger.info('NOOP: conf file contents would have been: \n%s' % my_template) # fake conf file name to return return '%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image) else: # conf file of the form /tmp_conf_dir/pool/imagename.conf try: conf_file = open( '%s/%s/%s.conf' % (settings.TEMP_CONF_DIR, pool, image), 'w') conf_file.write(my_template) except Exception as e: logger.error('error with temp conf file for orphan: %s error: %s' % (image, e)) raise return conf_file.name