def mktemp(self):
u"""
Return a unique filename suitable for use for a temporary
file. The file is not created.
Subsequent calls to this method are guaranteed to never return
the same filename again. As a result, it is safe to use under
concurrent conditions.
NOTE: mkstemp() is greatly preferred.
"""
filename = None
self.__lock.acquire()
try:
self.__tempcount = self.__tempcount + 1
suffix = u"-%d" % (self.__tempcount,)
filename = util.fsencode(tempfile.mktemp(suffix, u"mktemp-", self.__dir))
log.Debug(_(u"Registering (mktemp) temporary file %s") % util.fsdecode(filename))
self.__pending[filename] = None
finally:
self.__lock.release()
return filename
def sigtar2path_iter(sigtarobj):
u"""
Convert signature tar file object open for reading into path iter
"""
tf = util.make_tarfile(u"r", sigtarobj)
tf.debug = 1
for tarinfo in tf:
tiname = util.get_tarinfo_name(tarinfo)
for prefix in [r"signature/", r"snapshot/", r"deleted/"]:
if tiname.startswith(prefix):
# strip prefix and '/' from name and set it to difftype
name, difftype = tiname[len(prefix):], prefix[:-1]
break
else:
raise DiffDirException(u"Bad tarinfo name %s" % (tiname, ))
if sys.version_info.major >= 3:
index = tuple(util.fsencode(name).split(b"/"))
else:
index = tuple(name.split(b"/"))
if not index[-1]:
index = index[:-1] # deal with trailing /, ""
ropath = ROPath(index)
ropath.difftype = difftype
if difftype == u"signature" or difftype == u"snapshot":
ropath.init_from_tarinfo(tarinfo)
if ropath.isreg():
ropath.setfileobj(tf.extractfile(tarinfo))
yield ropath
sigtarobj.close()
def contains(self, child):
u"""Return true if path is a directory and contains child"""
if isinstance(child, u"".__class__):
child = util.fsencode(child)
# We don't use append(child).exists() here because that requires exec
# permissions as well as read. listdir() just needs read permissions.
return self.isdir() and child in self.listdir()
def transfer(self, method, source_path, remote_filename):
u"""create Par2 files and transfer the given file and the Par2 files
with the wrapped backend.
Par2 must run on the real filename or it would restore the
temp-filename later on. So first of all create a tempdir and symlink
the soure_path with remote_filename into this.
"""
par2temp = source_path.get_temp_in_same_dir()
par2temp.mkdir()
source_symlink = par2temp.append(remote_filename)
source_target = source_path.get_canonical()
if not os.path.isabs(source_target):
source_target = os.path.join(util.fsencode(os.getcwd()),
source_target)
os.symlink(source_target, source_symlink.get_canonical())
source_symlink.setdata()
log.Info(u"Create Par2 recovery files")
par2create = u'par2 c -r%d -n1 %s %s' % (
self.redundancy, self.common_options,
util.fsdecode(source_symlink.get_canonical()))
out, returncode = pexpect.run(par2create, None, True)
source_symlink.delete()
files_to_transfer = []
if not returncode:
for file in par2temp.listdir():
files_to_transfer.append(par2temp.append(file))
method(source_path, remote_filename)
for file in files_to_transfer:
method(file, file.get_filename())
par2temp.deltree()
def get_file_changed_record(self, filepath):
u"""
Returns time line of specified file changed
"""
# quick fix to spaces in filepath
modified_filepath = filepath
if u" " in filepath:
modified_filepath = u'"' + filepath.replace(u" ", r"\x20") + u'"'
if not self.matched_chain_pair:
return u""
all_backup_set = self.matched_chain_pair[1].get_all_sets()
specified_file_backup_set = []
specified_file_backup_type = []
modified_filepath = util.fsencode(modified_filepath)
for bs in all_backup_set:
filelist = [fileinfo[1] for fileinfo in bs.get_files_changed()]
if modified_filepath in filelist:
specified_file_backup_set.append(bs)
index = filelist.index(modified_filepath)
specified_file_backup_type.append(
bs.get_files_changed()[index][0])
return FileChangedStatus(
filepath,
list(zip(specified_file_backup_type, specified_file_backup_set)))
def _list(self):
commandline = u'%s "ls -l %s"' % (hsi_command, self.remote_dir)
l = self.subprocess_popen(commandline)[2]
l = l.split(os.linesep.encode())[3:]
for i in range(0, len(l)):
if l[i]:
l[i] = l[i].split()[-1]
return [util.fsencode(x) for x in l if x]
def tobytes(filename):
u"Convert a (maybe unicode) filename to bytes"
if isinstance(filename, str):
# There shouldn't be any encoding errors for files we care
# about, since duplicity filenames are ascii. But user files
# may be in the same directory. So just replace characters.
return util.fsencode(filename)
else:
return filename
def _list(self):
# Do a long listing to avoid connection reset
commandline = u"sxls {0}/".format(self.url_string)
_, l, _ = self.subprocess_popen(commandline)
# Look for our files as the last element of a long list line
return [
util.fsencode(x[x.rindex(u'/') + 1:].split()[-1])
for x in l.split(u'\n') if x and not x.startswith(u"total ")
]
def _list(self):
filename_list = []
for obj in self.bucket.objects.filter(Prefix=self.key_prefix):
try:
filename = obj.key.replace(self.key_prefix, u'', 1)
filename_list.append(util.fsencode(filename))
log.Debug(u"Listed %s/%s" % (self.straight_url, filename))
except AttributeError:
pass
return filename_list
def _list(self):
def split(str): # pylint: disable=redefined-builtin
line = str.split()
if len(line) > 4 and line[4] != u'.':
return line[4]
else:
return None
commandline = u"%s %s" % (self.cmd, self.url_string)
result, stdout, stderr = self.subprocess_popen(commandline)
return [util.fsencode(x) for x in map(split, stdout.split(u'\n')) if x]
def _list(self):
filelist = []
commandline = u"%s lsf '%s'" % (self.rclone_cmd, self.remote_path)
rc, o, e = self._subprocess_safe_popen(commandline)
if rc == 3:
return filelist
if rc != 0:
raise BackendException(e)
if not o:
return filelist
return [util.fsencode(x) for x in o.split(u'\n') if x]
def init_from_tarinfo(self, tarinfo):
u"""Set data from tarinfo object (part of tarfile module)"""
# Set the typepp
type = tarinfo.type # pylint: disable=redefined-builtin
if type == tarfile.REGTYPE or type == tarfile.AREGTYPE:
self.type = u"reg"
elif type == tarfile.LNKTYPE:
raise PathException(u"Hard links not supported yet")
elif type == tarfile.SYMTYPE:
self.type = u"sym"
self.symtext = tarinfo.linkname
if isinstance(self.symtext, u"".__class__):
self.symtext = util.fsencode(self.symtext)
elif type == tarfile.CHRTYPE:
self.type = u"chr"
self.devnums = (tarinfo.devmajor, tarinfo.devminor)
elif type == tarfile.BLKTYPE:
self.type = u"blk"
self.devnums = (tarinfo.devmajor, tarinfo.devminor)
elif type == tarfile.DIRTYPE:
self.type = u"dir"
elif type == tarfile.FIFOTYPE:
self.type = u"fifo"
else:
raise PathException(u"Unknown tarinfo type %s" % (type,))
self.mode = tarinfo.mode
self.stat = StatResult()
u""" If do_not_restore_owner is False,
set user and group id
use numeric id if name lookup fails
OR
--numeric-owner is set
"""
try:
if config.numeric_owner:
raise KeyError
self.stat.st_uid = cached_ops.getpwnam(tarinfo.uname)[2]
except KeyError:
self.stat.st_uid = tarinfo.uid
try:
if config.numeric_owner:
raise KeyError
self.stat.st_gid = cached_ops.getgrnam(tarinfo.gname)[2]
except KeyError:
self.stat.st_gid = tarinfo.gid
self.stat.st_mtime = int(tarinfo.mtime)
if self.stat.st_mtime < 0:
log.Warn(_(u"Warning: %s has negative mtime, treating as 0.")
% (tarinfo.uc_name))
self.stat.st_mtime = 0
self.stat.st_size = tarinfo.size
def rename_index(self, index):
if not config.rename or not index:
return index # early exit
path = os.path.normcase(os.path.join(*index))
tail = []
while path and path not in config.rename:
path, extra = os.path.split(path)
tail.insert(0, extra)
if path:
return config.rename[path].split(util.fsencode(os.sep)) + tail
else:
return index # no rename found
def delete(self, filename):
u"""delete given filename and its .par2 files
"""
self.wrapped_backend._delete(filename)
remote_list = self.unfiltered_list()
c = re.compile(u'%s(?:\\.vol[\\d+]*)?\\.par2' %
util.fsdecode(filename))
for remote_filename in remote_list:
if c.match(util.fsdecode(remote_filename)):
self.wrapped_backend._delete(util.fsencode(remote_filename))
def _list(self):
# Do a long listing to avoid connection reset
# remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
remote_dir = urllib.parse.unquote(self.parsed_url.path)
# print remote_dir
quoted_path = cmd_quote(self.remote_path)
# failing to cd into the folder might be because it was not created already
commandline = u"lftp -c \"source %s; ( cd %s && ls ) || ( mkdir -p %s && cd %s && ls )\"" % (
cmd_quote(self.tempname), quoted_path, quoted_path, quoted_path)
log.Debug(u"CMD: %s" % commandline)
_, l, e = self.subprocess_popen(commandline)
log.Debug(u"STDERR:\n" u"%s" % (e))
log.Debug(u"STDOUT:\n" u"%s" % (l))
# Look for our files as the last element of a long list line
return [util.fsencode(x.split()[-1]) for x in l.split(u'\n') if x]
def get_index_from_tarinfo(tarinfo):
u"""Return (index, difftype, multivol) pair from tarinfo object"""
for prefix in [
u"snapshot/", u"diff/", u"deleted/", u"multivol_diff/",
u"multivol_snapshot/"
]:
tiname = util.get_tarinfo_name(tarinfo)
if sys.version_info.major == 2 and isinstance(prefix, unicode):
prefix = prefix.encode()
if tiname.startswith(prefix):
name = tiname[len(prefix):] # strip prefix
if prefix.startswith(u"multivol"):
if prefix == u"multivol_diff/":
difftype = u"diff"
else:
difftype = u"snapshot"
multivol = 1
name, num_subs = \
re.subn(u"(?s)^multivol_(diff|snapshot)/?(.*)/[0-9]+$",
u"\\2", tiname)
if num_subs != 1:
raise PatchDirException(u"Unrecognized diff entry %s" %
tiname)
else:
difftype = prefix[:-1] # strip trailing /
name = tiname[len(prefix):]
if name.endswith(r"/"):
name = name[:-1] # strip trailing /'s
multivol = 0
break
else:
raise PatchDirException(u"Unrecognized diff entry %s" % tiname)
if name == r"." or name == r"":
index = ()
else:
if sys.version_info.major >= 3:
index = tuple(util.fsencode(name).split(b"/"))
else:
index = tuple(name.split(b"/"))
if b'..' in index:
raise PatchDirException(u"Tar entry %s contains '..'. Security "
u"violation" % util.fsdecode(tiname))
return (index, difftype, multivol)
def __init__(self, base, index=()):
u"""Path initializer"""
# self.opened should be true if the file has been opened, and
# self.fileobj can override returned fileobj
self.opened, self.fileobj = None, None
if isinstance(base, str):
# For now (Python 2), it is helpful to know that all paths
# are starting with bytes -- see note above util.fsencode definition
base = util.fsencode(base)
self.base = base
# Create self.index, which is the path as a tuple
self.index = self.rename_index(index)
self.name = os.path.join(base, *self.index)
# We converted any unicode base to filesystem encoding, so self.name should
# be in filesystem encoding already and does not need to change
self.uc_name = util.fsdecode(self.name)
self.setdata()
def __init__(self, base, index=()):
"""Path initializer"""
# self.opened should be true if the file has been opened, and
# self.fileobj can override returned fileobj
self.opened, self.fileobj = None, None
if isinstance(base, unicode):
# For now (Python 2), it is helpful to know that all paths
# are starting with bytes -- see note above util.fsencode definition
base = util.fsencode(base)
self.base = base
# Create self.index, which is the path as a tuple
self.index = self.rename_index(index)
self.name = os.path.join(base, *self.index)
# We converted any unicode base to filesystem encoding, so self.name should
# be in filesystem encoding already and does not need to change
self.uc_name = util.fsdecode(self.name)
self.setdata()
def folder_contents(self, files_only=False):
u'lists contents of a folder, optionally ignoring subdirectories'
print(u"megals: %s" % (self._folder, ))
if self._megarc:
cmd = [u'megals', u'--config', self._megarc, self._folder]
else:
cmd = [
u'megals', u'-u', self._username, u'-p', self._password,
self._folder
]
files = subprocess.check_output(cmd)
files = util.fsdecode(files.strip()).split(u'\n')
# remove the folder name, including the path separator
files = [f[len(self._folder) + 1:] for f in files]
# optionally ignore entries containing path separator (i.e. not files)
if files_only:
files = [f for f in files if u'/' not in f]
return [util.fsencode(f) for f in files]
def compare_verbose(self, other, include_data=0):
u"""Compare ROPaths like __eq__, but log reason if different
This is placed in a separate function from __eq__ because
__eq__ should be very time sensitive, and logging statements
would slow it down. Used when verifying.
Only run if include_data is true.
"""
def log_diff(log_string):
log_str = _(u"Difference found:") + u" " + log_string
log.Notice(log_str % (util.fsdecode(self.get_relative_path())))
if include_data is False:
return True
if not self.type and not other.type:
return 1
if not self.stat and other.stat:
log_diff(_(u"New file %s"))
return 0
if not other.stat and self.stat:
log_diff(_(u"File %s is missing"))
return 0
if self.type != other.type:
log_diff(_(u"File %%s has type %s, expected %s") %
(other.type, self.type))
return 0
if self.isreg() or self.isdir() or self.isfifo():
if not self.perms_equal(other):
log_diff(_(u"File %%s has permissions %s, expected %s") %
(other.getperms(), self.getperms()))
return 0
if ((int(self.stat.st_mtime) != int(other.stat.st_mtime)) and
(self.stat.st_mtime > 0 or other.stat.st_mtime > 0)):
log_diff(_(u"File %%s has mtime %s, expected %s") %
(dup_time.timetopretty(int(other.stat.st_mtime)),
dup_time.timetopretty(int(self.stat.st_mtime))))
return 0
if self.isreg():
if self.compare_data(other):
return 1
else:
log_diff(_(u"Data for file %s is different"))
return 0
else:
return 1
elif self.issym():
if self.symtext == other.symtext or self.symtext + util.fsencode(os.sep) == other.symtext:
return 1
else:
log_diff(_(u"Symlink %%s points to %s, expected %s") %
(other.symtext, self.symtext))
return 0
elif self.isdev():
if not self.perms_equal(other):
log_diff(_(u"File %%s has permissions %s, expected %s") %
(other.getperms(), self.getperms()))
return 0
if self.devnums != other.devnums:
log_diff(_(u"Device file %%s has numbers %s, expected %s")
% (other.devnums, self.devnums))
return 0
return 1
assert 0
def append(self, ext):
u"""Return new Path with ext added to index"""
if isinstance(ext, u"".__class__):
ext = util.fsencode(ext)
return self.__class__(self.base, self.index + (ext, ))
def _list(self):
return [util.fsencode(o[u'name'][len(self.prefix):]) for o in self.__list_objs()]
def _list(self):
headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix)
# removes prefix from return values. should check for the prefix ?
return [util.fsencode(o[u'name'][len(self.prefix):]) for o in objs]
def _list(self):
# Do a long listing to avoid connection reset
commandline = u"ncftpls %s -l '%s'" % (self.flags, self.url_string)
_, l, _ = self.subprocess_popen(commandline)
# Look for our files as the last element of a long list line
return [util.fsencode(x.split()[-1]) for x in l.split(u'\n') if x and not x.startswith(u"total ")]
def _list(self):
output = self.run(u"tahoe", u"ls", self.get_remote_path())
return [util.fsencode(x) for x in output.split(u'\n') if x]
