def commitflistFiles(self):
"""
Commit the include.list and exclude.list to the disk.
@todo: simplify the written files! The *.tmp files are partly obsolete.
"""
if self._fop.path_exists(self.getIncludeFListFile()):
raise SBException("includes.list should not exist at this stage")
if self._fop.path_exists(self.getExcludeFListFile()):
raise SBException("excludes.list should not exist at this stage")
self._fop.writetofile(self.getIncludeFListFile(),
"\n".join(self.__includeFlist.getEffectiveFileList()))
self._fop.writetofile(self.getExcludeFListFile(),
"\n".join(self.__excludeFlist.getEffectiveFileList()))
# commit temporary lists
#FIXME: unify creation and storage of tmp. files and names
tmp_incl = self._fop.normpath(ConfigurationFileHandler().get_user_tempdir(),
self._fop.get_basename(self.getIncludeFListFile()))
tmp_excl = self._fop.normpath(ConfigurationFileHandler().get_user_tempdir(),
self._fop.get_basename(self.getExcludeFListFile()))
self._fop.writetofile(tmp_incl,
"\n".join(self.__includeFlist.get_eff_filelist_not_nested()))
self._fop.writetofile(tmp_excl,
"\n".join(self.__excludeFlist.getEffectiveFileList()))
def getSnpHistory(self, snapshot):
"""
gets the list of preceding snapshots till the last full one
:param snapshot : the given snapshot
:return: a list of Snapshots starting from the most recent one to the full one
:note: you'll need to reverse this list to make a revert
"""
if not snapshot:
raise SBException("Please provide a snapshot to process")
result = []
# add the first snapshot
result.append(snapshot)
current = snapshot
while (current.getBaseSnapshot()):
current = current.getBaseSnapshot()
result.append(current)
# Just for DEBUG
if self.logger.isEnabledFor(10):
# get the history
history = "\n[%s history]" % snapshot.getName()
for snp in result:
history += "\n- %s" % snp.getName()
self.logger.debug(history)
return result
def getVersion(self):
"""Retrieves and returns the version of the snapshot.
"""
if self.__version:
return self.__version
elif ":" in self.getName():
self.__version = "1.0"
return self.__version
else:
verfile = self._fop.joinpath(self.getPath(), "ver")
if not self._fop.path_exists(verfile):
return False
else:
ver = self._fop.readfile(verfile)
# self.logger.debug("Version read from snapshot: `%s`" % ver)
try:
# major =
int(ver[0])
# minor =
int(ver[2])
except Exception:
raise SBException(_("%(file)s doesn't contain valid value. Ignoring incomplete or non - backup directory. ") % {"file" : verfile})
self.__version = ver[:3]
return self.__version
def getPath(self):
"return the complete path of the snapshot"
if not self.__snapshotpath:
raise SBException(
_("Snapshot is inconsistent: __snapshotpath is not set "))
else:
return self.__snapshotpath
def getName(self):
" return the name of the snapshot (ie the dir name)"
if not self.__name:
raise SBException(
_("Snapshot is inconsistent: __name is not set "))
else:
return self.__name
def _copy_empty_snar(self, snp_source, copydest):
"""Creates an empty SnapshotInfo-file with the name 'copydest'
from the SnapshotInfo-file contained in given source snapshot. Empty
means, that no content but the header is copied.
@todo: Review the self-creation of header in the case no was found.
Is this necessary at all?
"""
self.logger.debug("Create temporary SNARFILE to prepare merging")
if not isinstance(snp_source, Snapshot):
raise TypeError("Given parameter 'snp_source' must be of Snapshot "\
"type! Got %s instead." % type(snp_source))
if not isinstance(copydest, str):
raise TypeError("Given parameter 'copydest' must be of string "\
"type! Got %s instead." % type(copydest))
# create a temporary snar file for merge result
_tmpfinal = copydest
# get snar header from current snapshots
#XXX: Why not use getHeader here???
_snarf = open(snp_source.getSnarFile())
_header = _snarf.readline()
if len(_header) > 0:
# the SNAR file isn't empty
sepcnt = 0
while sepcnt < 2:
readchar = _snarf.read(1)
if len(readchar) != 1:
_snarf.close()
raise SBException(_("The snarfile header is incomplete !"))
if readchar == '\0':
sepcnt += 1
_header += readchar
_snarf.close()
self.logger.debug("Current SNAR Header (NULL replaced by newline):"\
"\n%s" % (_header.replace("\x00", "\n")))
else:
# the SNAR file is empty
self.logger.debug("SNAR file empty, create the header manually")
_snarf.close()
_date = snp_source.getDate()
_datet = datetime.datetime(_date['year'], _date['month'],
_date['day'], _date['hour'],
_date['minute'], _date['second'])
# create temporary SNAR file and copy the retrieved header into it
finalsnar = ProcSnapshotFile(SnapshotFile(_tmpfinal, True))
if _header:
snpif = open(_tmpfinal, 'w')
snpif.write(_header)
snpif.close()
else:
finalsnar.setHeader(_datet)
_header = finalsnar.getHeader()
return finalsnar
def __check_mountdir(self):
"""check if the mount dir is valid
:todo: Do not create dir here? Use makedirs?
"""
if not os.path.exists(self.__mountdir):
os.mkdir(self.__mountdir)
else:
if not os.path.isdir(self.__mountdir):
raise SBException("The mount base dir should be a directory")
def setBase(self, baseName):
"""Sets `baseName` as the name of the base snapshot of this
snapshot and clears the reference to base snapshot object.
It is not possible to set the base for a full backup, this
raises a `SBException`. The `baseName` is checked for validity.
Note that the base of the snapshot is not committed to disk
if it is set using this method. Call `commitbasefile` for this.
"""
if self.isfull():
self.__base = None
self.__baseSnapshot = None
raise SBException("Base cannot be set for full snapshot.")
if not self.__isValidName(baseName):
raise SBException (_("Name of base not valid : %s") % self.__name)
# set the name and clean the baseSnapshot
self.__base = baseName
self.__baseSnapshot = None
def getSon(self, path):
"""
get the son SBdict of a path
@param path: the path to get the son of
@return: a SBdict or None if there were no son
"""
if path in self:
return self[path][1]
else:
raise SBException("'%s' not in SBdict" % path)
def get_snapshot_allformats(self, name):
"""Returns a certain snapshot, specified by its name, from the stored
snapshots. If the snapshot could not be found, an exception is raised.
:param name: the snapshot that is to be returned
"""
for snp in self.get_snapshots_allformats():
if snp.getName() == name:
return snp
raise SBException(_("Snapshot '%s' not found ") % name)
def commitbasefile(self):
"""In case this snapshot is an incremental snapshot, base file is
committed to the disk. If not, this method shouldn't be called.
The absence of a base for an incremental backup raises a SBException.
"""
if self.isfull():
self.logger.debug("WARNING: Attempt of committing base file for "\
"full snapshot ' % s'." % self.getName())
else:
if self.getBase():
_basef = self._fop.joinpath(self.getPath(), "base")
self._fop.writetofile(_basef, self.getBase())
else:
# base file was not found or base wasn't set. It MUST be full backup
raise SBException(_("Base name must be set for incremental backup."))
def iterFirstItems(self, _path=None):
"""
an Iterator that gets recursively the path off first items
Should return props
"""
if _path is None: # initialization
_path = []
for dirname, (props, son) in dict.items(self):
_path.append(dirname)
if props != None:
yield os.sep.join(_path)
_path.pop()
else:
if son is not None:
for path in son.iterFirstItems(_path):
yield path
else:
raise SBException(
"getting to an ending file without properties")
if len(_path) > 0:
_path.pop()
def _merge_snarfiles(self, target_snpfinfo, target_excludes, src_snpfinfo,
res_snpfinfo):
"""Covers all actions for merging 2 given snar files into a single
one. This is quite TAR specific - think it over where to place it!
:Parameters:
- `target_snpfinfo`: the resulting snapshot
- `target_excludes`: set of the excludes file list of resulting snapshot
- `src_snpfinfo`: the snapshot that should be merged into the target
- `res_snpfinfo`: the name of the resulting SNAR file
The method returns a list containing files that needs to be extracted
from the archive that was merged in.
:todo: Do we need to consider the order of the snar files?
:todo: Needs more refactoring! (CQS)
"""
self.logger.info("Merging SNARFILEs to make the transfer")
if not isinstance(target_snpfinfo, SnapshotFileWrapper):
raise TypeError("Given parameter 'target_snpfinfo' must be of "\
"SnapshotFileWrapper "\
"type! Got %s instead." % type(target_snpfinfo))
if not isinstance(target_excludes, set):
raise TypeError("Given parameter 'target_excludes' must be of "\
"type Set! "\
"Got %s instead." % type(target_excludes))
if not isinstance(src_snpfinfo, SnapshotFileWrapper):
raise TypeError("Given parameter 'src_snpfinfo' must be of "\
"SnapshotFileWrapper "\
"type! Got %s instead." % type(src_snpfinfo))
if not isinstance(res_snpfinfo, SnapshotFileWrapper):
raise TypeError("Given parameter 'res_snpfinfo' must be of "\
"SnapshotFileWrapper "\
"type! Got %s instead." % type(res_snpfinfo))
# print("Parent (base) snar file:\n%s" % src_snpfinfo)
# list for storage of files that need to be extracted from merge source
files_to_extract = []
for target_record in target_snpfinfo.iterRecords():
_tmp_dumpdirs = []
#TODO: A similar method to getContent would be nice!
_curdir = target_record[SnapshotFile.REC_DIRNAME]
# get the content (dumpdir entries) for current directory
_curcontent = target_snpfinfo.getContent(_curdir)
for _dumpdir in _curcontent:
# print("\n now processing dumpdir: %s" % _dumpdir)
_ctrl = _dumpdir.getControl()
_filen = _dumpdir.getFilename()
_ddir_final = None
_was_excluded = False
if _ctrl == Dumpdir.UNCHANGED:
# Item was explicitly excluded and is therefore not included in child
# _filenfull = os.path.join(_curdir, _filen)
# print("Full path: %s" % (_filenfull))
if self._fop.joinpath(_curdir, _filen) in target_excludes:
self.logger.debug(
"Path '%s' was excluded. Not merged." % _filen)
_was_excluded = True
else:
# Item has not changed and is therefore not included in child (i.e. target) snapshot.
# look for the item in the parent (i.e. base/source) snapshot
_basedumpd = get_dumpdir_from_list(\
src_snpfinfo.getContent(_curdir),
_filen)
_base_ctrl = _basedumpd.getControl()
if _base_ctrl == Dumpdir.UNCHANGED:
_ddir_final = _dumpdir
elif _base_ctrl == Dumpdir.INCLUDED:
_ddir_final = _basedumpd
files_to_extract.append(
self._fop.joinpath(_curdir, _filen))
else:
raise SBException("Found unexpected control code "\
"('%s') in snapshot file '%s'."\
% (_ctrl, target_snpfinfo.get_snapfile_path()))
elif _ctrl == Dumpdir.DIRECTORY:
_ddir_final = _dumpdir
elif _ctrl == Dumpdir.INCLUDED:
_ddir_final = _dumpdir
else:
raise SBException("Found unexpected control code "\
"('%s') in snapshot file '%s'."\
% (_ctrl, target_snpfinfo.get_snpfile_Path()))
if not _was_excluded:
_tmp_dumpdirs.append(_ddir_final)
# end of loop over dumpdirs
_final_record = target_record[:SnapshotFile.REC_CONTENT]
_final_record.append(_tmp_dumpdirs)
# write to the SnarFile
res_snpfinfo.addRecord(_final_record)
return files_to_extract
def mount(self, source, mountbase):
"""
Mount the source intor the mountbase dir . This method should create a mount point to mount the source.
The name of the mount point should be very expressive so that we avoid collision with other mount points
@param source: The remote path
@param mountbase: The mount points base dir
@return: The mount point complete path
"""
exp = re.compile(ssh_url_re)
match = exp.search(source)
if not match:
raise FuseFAMException(
_("Error matching the schema 'ssh://*****:*****@example.com/home/' with '%s' (The '/' after server is mandatory)"
) % source)
else:
remoteSource = "ssh://" + match.group(1)
if match.group(3):
remoteSource += ":" + match.group(3)
remoteSource += "@" + match.group(4)
if match.group(6):
remoteSource += ":" + match.group(6)
remoteSource += "/"
user = match.group(1)
mountpoint = local_file_utils.joinpath(
mountbase, self._defineMountDirName(source))
if match.group(7):
pathinside = match.group(7)
else:
pathinside = ""
#If the path is already mounted No need to retry
if self.checkifmounted(source, mountbase):
return (remoteSource, mountpoint, pathinside)
cmd = "sshfs " + user + "@" + match.group(4) + ":/"
cmd = cmd + " " + mountpoint
port = match.group(6)
if port:
cmd += " -p " + port
if not local_file_utils.path_exists(mountpoint):
local_file_utils.makedir(mountpoint)
if system.is_superuser():
cmd += " -o allow_root"
self.logger.debug("Spawning: " + cmd)
password = match.group(3)
sshfsp = pexpect.spawn(cmd)
i = sshfsp.expect(['(yes/no)', 'password:'******'Password:'******'yes')
i = sshfsp.expect(
['(yes/no)', 'password:'******'Password:'******'(yes/no)', 'password:'******'Password:'******'%(command)s' didn't perform normally. Output => %(erroroutput)s "
) % {
"command": cmd,
"erroroutput": result
})
return (remoteSource, mountpoint, pathinside)