def mktemp(self): u""" Return a unique filename suitable for use for a temporary file. The file is not created. Subsequent calls to this method are guaranteed to never return the same filename again. As a result, it is safe to use under concurrent conditions. NOTE: mkstemp() is greatly preferred. """ filename = None self.__lock.acquire() try: self.__tempcount = self.__tempcount + 1 suffix = u"-%d" % (self.__tempcount,) filename = util.fsencode(tempfile.mktemp(suffix, u"mktemp-", self.__dir)) log.Debug(_(u"Registering (mktemp) temporary file %s") % util.fsdecode(filename)) self.__pending[filename] = None finally: self.__lock.release() return filename
def sigtar2path_iter(sigtarobj): u""" Convert signature tar file object open for reading into path iter """ tf = util.make_tarfile(u"r", sigtarobj) tf.debug = 1 for tarinfo in tf: tiname = util.get_tarinfo_name(tarinfo) for prefix in [r"signature/", r"snapshot/", r"deleted/"]: if tiname.startswith(prefix): # strip prefix and '/' from name and set it to difftype name, difftype = tiname[len(prefix):], prefix[:-1] break else: raise DiffDirException(u"Bad tarinfo name %s" % (tiname, )) if sys.version_info.major >= 3: index = tuple(util.fsencode(name).split(b"/")) else: index = tuple(name.split(b"/")) if not index[-1]: index = index[:-1] # deal with trailing /, "" ropath = ROPath(index) ropath.difftype = difftype if difftype == u"signature" or difftype == u"snapshot": ropath.init_from_tarinfo(tarinfo) if ropath.isreg(): ropath.setfileobj(tf.extractfile(tarinfo)) yield ropath sigtarobj.close()
def contains(self, child): u"""Return true if path is a directory and contains child""" if isinstance(child, u"".__class__): child = util.fsencode(child) # We don't use append(child).exists() here because that requires exec # permissions as well as read. listdir() just needs read permissions. return self.isdir() and child in self.listdir()
def transfer(self, method, source_path, remote_filename): u"""create Par2 files and transfer the given file and the Par2 files with the wrapped backend. Par2 must run on the real filename or it would restore the temp-filename later on. So first of all create a tempdir and symlink the soure_path with remote_filename into this. """ par2temp = source_path.get_temp_in_same_dir() par2temp.mkdir() source_symlink = par2temp.append(remote_filename) source_target = source_path.get_canonical() if not os.path.isabs(source_target): source_target = os.path.join(util.fsencode(os.getcwd()), source_target) os.symlink(source_target, source_symlink.get_canonical()) source_symlink.setdata() log.Info(u"Create Par2 recovery files") par2create = u'par2 c -r%d -n1 %s %s' % ( self.redundancy, self.common_options, util.fsdecode(source_symlink.get_canonical())) out, returncode = pexpect.run(par2create, None, True) source_symlink.delete() files_to_transfer = [] if not returncode: for file in par2temp.listdir(): files_to_transfer.append(par2temp.append(file)) method(source_path, remote_filename) for file in files_to_transfer: method(file, file.get_filename()) par2temp.deltree()
def get_file_changed_record(self, filepath): u""" Returns time line of specified file changed """ # quick fix to spaces in filepath modified_filepath = filepath if u" " in filepath: modified_filepath = u'"' + filepath.replace(u" ", r"\x20") + u'"' if not self.matched_chain_pair: return u"" all_backup_set = self.matched_chain_pair[1].get_all_sets() specified_file_backup_set = [] specified_file_backup_type = [] modified_filepath = util.fsencode(modified_filepath) for bs in all_backup_set: filelist = [fileinfo[1] for fileinfo in bs.get_files_changed()] if modified_filepath in filelist: specified_file_backup_set.append(bs) index = filelist.index(modified_filepath) specified_file_backup_type.append( bs.get_files_changed()[index][0]) return FileChangedStatus( filepath, list(zip(specified_file_backup_type, specified_file_backup_set)))
def _list(self): commandline = u'%s "ls -l %s"' % (hsi_command, self.remote_dir) l = self.subprocess_popen(commandline)[2] l = l.split(os.linesep.encode())[3:] for i in range(0, len(l)): if l[i]: l[i] = l[i].split()[-1] return [util.fsencode(x) for x in l if x]
def tobytes(filename): u"Convert a (maybe unicode) filename to bytes" if isinstance(filename, str): # There shouldn't be any encoding errors for files we care # about, since duplicity filenames are ascii. But user files # may be in the same directory. So just replace characters. return util.fsencode(filename) else: return filename
def _list(self): # Do a long listing to avoid connection reset commandline = u"sxls {0}/".format(self.url_string) _, l, _ = self.subprocess_popen(commandline) # Look for our files as the last element of a long list line return [ util.fsencode(x[x.rindex(u'/') + 1:].split()[-1]) for x in l.split(u'\n') if x and not x.startswith(u"total ") ]
def _list(self): filename_list = [] for obj in self.bucket.objects.filter(Prefix=self.key_prefix): try: filename = obj.key.replace(self.key_prefix, u'', 1) filename_list.append(util.fsencode(filename)) log.Debug(u"Listed %s/%s" % (self.straight_url, filename)) except AttributeError: pass return filename_list
def _list(self): def split(str): # pylint: disable=redefined-builtin line = str.split() if len(line) > 4 and line[4] != u'.': return line[4] else: return None commandline = u"%s %s" % (self.cmd, self.url_string) result, stdout, stderr = self.subprocess_popen(commandline) return [util.fsencode(x) for x in map(split, stdout.split(u'\n')) if x]
def _list(self): filelist = [] commandline = u"%s lsf '%s'" % (self.rclone_cmd, self.remote_path) rc, o, e = self._subprocess_safe_popen(commandline) if rc == 3: return filelist if rc != 0: raise BackendException(e) if not o: return filelist return [util.fsencode(x) for x in o.split(u'\n') if x]
def init_from_tarinfo(self, tarinfo): u"""Set data from tarinfo object (part of tarfile module)""" # Set the typepp type = tarinfo.type # pylint: disable=redefined-builtin if type == tarfile.REGTYPE or type == tarfile.AREGTYPE: self.type = u"reg" elif type == tarfile.LNKTYPE: raise PathException(u"Hard links not supported yet") elif type == tarfile.SYMTYPE: self.type = u"sym" self.symtext = tarinfo.linkname if isinstance(self.symtext, u"".__class__): self.symtext = util.fsencode(self.symtext) elif type == tarfile.CHRTYPE: self.type = u"chr" self.devnums = (tarinfo.devmajor, tarinfo.devminor) elif type == tarfile.BLKTYPE: self.type = u"blk" self.devnums = (tarinfo.devmajor, tarinfo.devminor) elif type == tarfile.DIRTYPE: self.type = u"dir" elif type == tarfile.FIFOTYPE: self.type = u"fifo" else: raise PathException(u"Unknown tarinfo type %s" % (type,)) self.mode = tarinfo.mode self.stat = StatResult() u""" If do_not_restore_owner is False, set user and group id use numeric id if name lookup fails OR --numeric-owner is set """ try: if config.numeric_owner: raise KeyError self.stat.st_uid = cached_ops.getpwnam(tarinfo.uname)[2] except KeyError: self.stat.st_uid = tarinfo.uid try: if config.numeric_owner: raise KeyError self.stat.st_gid = cached_ops.getgrnam(tarinfo.gname)[2] except KeyError: self.stat.st_gid = tarinfo.gid self.stat.st_mtime = int(tarinfo.mtime) if self.stat.st_mtime < 0: log.Warn(_(u"Warning: %s has negative mtime, treating as 0.") % (tarinfo.uc_name)) self.stat.st_mtime = 0 self.stat.st_size = tarinfo.size
def rename_index(self, index): if not config.rename or not index: return index # early exit path = os.path.normcase(os.path.join(*index)) tail = [] while path and path not in config.rename: path, extra = os.path.split(path) tail.insert(0, extra) if path: return config.rename[path].split(util.fsencode(os.sep)) + tail else: return index # no rename found
def delete(self, filename): u"""delete given filename and its .par2 files """ self.wrapped_backend._delete(filename) remote_list = self.unfiltered_list() c = re.compile(u'%s(?:\\.vol[\\d+]*)?\\.par2' % util.fsdecode(filename)) for remote_filename in remote_list: if c.match(util.fsdecode(remote_filename)): self.wrapped_backend._delete(util.fsencode(remote_filename))
def _list(self): # Do a long listing to avoid connection reset # remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip() remote_dir = urllib.parse.unquote(self.parsed_url.path) # print remote_dir quoted_path = cmd_quote(self.remote_path) # failing to cd into the folder might be because it was not created already commandline = u"lftp -c \"source %s; ( cd %s && ls ) || ( mkdir -p %s && cd %s && ls )\"" % ( cmd_quote(self.tempname), quoted_path, quoted_path, quoted_path) log.Debug(u"CMD: %s" % commandline) _, l, e = self.subprocess_popen(commandline) log.Debug(u"STDERR:\n" u"%s" % (e)) log.Debug(u"STDOUT:\n" u"%s" % (l)) # Look for our files as the last element of a long list line return [util.fsencode(x.split()[-1]) for x in l.split(u'\n') if x]
def get_index_from_tarinfo(tarinfo): u"""Return (index, difftype, multivol) pair from tarinfo object""" for prefix in [ u"snapshot/", u"diff/", u"deleted/", u"multivol_diff/", u"multivol_snapshot/" ]: tiname = util.get_tarinfo_name(tarinfo) if sys.version_info.major == 2 and isinstance(prefix, unicode): prefix = prefix.encode() if tiname.startswith(prefix): name = tiname[len(prefix):] # strip prefix if prefix.startswith(u"multivol"): if prefix == u"multivol_diff/": difftype = u"diff" else: difftype = u"snapshot" multivol = 1 name, num_subs = \ re.subn(u"(?s)^multivol_(diff|snapshot)/?(.*)/[0-9]+$", u"\\2", tiname) if num_subs != 1: raise PatchDirException(u"Unrecognized diff entry %s" % tiname) else: difftype = prefix[:-1] # strip trailing / name = tiname[len(prefix):] if name.endswith(r"/"): name = name[:-1] # strip trailing /'s multivol = 0 break else: raise PatchDirException(u"Unrecognized diff entry %s" % tiname) if name == r"." or name == r"": index = () else: if sys.version_info.major >= 3: index = tuple(util.fsencode(name).split(b"/")) else: index = tuple(name.split(b"/")) if b'..' in index: raise PatchDirException(u"Tar entry %s contains '..'. Security " u"violation" % util.fsdecode(tiname)) return (index, difftype, multivol)
def __init__(self, base, index=()): u"""Path initializer""" # self.opened should be true if the file has been opened, and # self.fileobj can override returned fileobj self.opened, self.fileobj = None, None if isinstance(base, str): # For now (Python 2), it is helpful to know that all paths # are starting with bytes -- see note above util.fsencode definition base = util.fsencode(base) self.base = base # Create self.index, which is the path as a tuple self.index = self.rename_index(index) self.name = os.path.join(base, *self.index) # We converted any unicode base to filesystem encoding, so self.name should # be in filesystem encoding already and does not need to change self.uc_name = util.fsdecode(self.name) self.setdata()
def __init__(self, base, index=()): """Path initializer""" # self.opened should be true if the file has been opened, and # self.fileobj can override returned fileobj self.opened, self.fileobj = None, None if isinstance(base, unicode): # For now (Python 2), it is helpful to know that all paths # are starting with bytes -- see note above util.fsencode definition base = util.fsencode(base) self.base = base # Create self.index, which is the path as a tuple self.index = self.rename_index(index) self.name = os.path.join(base, *self.index) # We converted any unicode base to filesystem encoding, so self.name should # be in filesystem encoding already and does not need to change self.uc_name = util.fsdecode(self.name) self.setdata()
def folder_contents(self, files_only=False): u'lists contents of a folder, optionally ignoring subdirectories' print(u"megals: %s" % (self._folder, )) if self._megarc: cmd = [u'megals', u'--config', self._megarc, self._folder] else: cmd = [ u'megals', u'-u', self._username, u'-p', self._password, self._folder ] files = subprocess.check_output(cmd) files = util.fsdecode(files.strip()).split(u'\n') # remove the folder name, including the path separator files = [f[len(self._folder) + 1:] for f in files] # optionally ignore entries containing path separator (i.e. not files) if files_only: files = [f for f in files if u'/' not in f] return [util.fsencode(f) for f in files]
def compare_verbose(self, other, include_data=0): u"""Compare ROPaths like __eq__, but log reason if different This is placed in a separate function from __eq__ because __eq__ should be very time sensitive, and logging statements would slow it down. Used when verifying. Only run if include_data is true. """ def log_diff(log_string): log_str = _(u"Difference found:") + u" " + log_string log.Notice(log_str % (util.fsdecode(self.get_relative_path()))) if include_data is False: return True if not self.type and not other.type: return 1 if not self.stat and other.stat: log_diff(_(u"New file %s")) return 0 if not other.stat and self.stat: log_diff(_(u"File %s is missing")) return 0 if self.type != other.type: log_diff(_(u"File %%s has type %s, expected %s") % (other.type, self.type)) return 0 if self.isreg() or self.isdir() or self.isfifo(): if not self.perms_equal(other): log_diff(_(u"File %%s has permissions %s, expected %s") % (other.getperms(), self.getperms())) return 0 if ((int(self.stat.st_mtime) != int(other.stat.st_mtime)) and (self.stat.st_mtime > 0 or other.stat.st_mtime > 0)): log_diff(_(u"File %%s has mtime %s, expected %s") % (dup_time.timetopretty(int(other.stat.st_mtime)), dup_time.timetopretty(int(self.stat.st_mtime)))) return 0 if self.isreg(): if self.compare_data(other): return 1 else: log_diff(_(u"Data for file %s is different")) return 0 else: return 1 elif self.issym(): if self.symtext == other.symtext or self.symtext + util.fsencode(os.sep) == other.symtext: return 1 else: log_diff(_(u"Symlink %%s points to %s, expected %s") % (other.symtext, self.symtext)) return 0 elif self.isdev(): if not self.perms_equal(other): log_diff(_(u"File %%s has permissions %s, expected %s") % (other.getperms(), self.getperms())) return 0 if self.devnums != other.devnums: log_diff(_(u"Device file %%s has numbers %s, expected %s") % (other.devnums, self.devnums)) return 0 return 1 assert 0
def append(self, ext): u"""Return new Path with ext added to index""" if isinstance(ext, u"".__class__): ext = util.fsencode(ext) return self.__class__(self.base, self.index + (ext, ))
def _list(self): return [util.fsencode(o[u'name'][len(self.prefix):]) for o in self.__list_objs()]
def _list(self): headers, objs = self.conn.get_container(self.container, full_listing=True, path=self.prefix) # removes prefix from return values. should check for the prefix ? return [util.fsencode(o[u'name'][len(self.prefix):]) for o in objs]
def _list(self): # Do a long listing to avoid connection reset commandline = u"ncftpls %s -l '%s'" % (self.flags, self.url_string) _, l, _ = self.subprocess_popen(commandline) # Look for our files as the last element of a long list line return [util.fsencode(x.split()[-1]) for x in l.split(u'\n') if x and not x.startswith(u"total ")]
def _list(self): output = self.run(u"tahoe", u"ls", self.get_remote_path()) return [util.fsencode(x) for x in output.split(u'\n') if x]