def TarInfo_str(tarinfo, list_in_bytes = False):
'''Creates a human-readable string from tarfile.TarInfo object
:param tarinfo: tarfile.TarInfo, the tarfile member to stringify
:return: str, Human-readable string of the tarfile member object
Repackaged parts of the original tarfile code (c.f. tarfile.list method).
'''
assert(isinstance(tarinfo, tarfile.TarInfo))
name_suffix = ''
if tarinfo.isdir():
name_suffix += os.path.sep
if tarinfo.issym():
name_suffix += ' -> ' + tarinfo.linkname
if tarinfo.islnk():
name_suffix += ' link to ' + tarinfo.linkname
if list_in_bytes:
tarinfo_size = str(tarinfo.size).rjust(12)
else:
tarinfo_size = sz_str(tarinfo.size).rjust(8)
tarinfo_str = "%s %s %s %s %s" % (
tarfile.filemode(tarinfo.mode),
tarinfo_size,
TarInfo_type(tarinfo.type).rjust(5),
"%d-%02d-%02d %02d:%02d:%02d" % time.localtime(tarinfo.mtime)[:6],
tarinfo.name + name_suffix,
)
return tarinfo_str
def get_tarinfos(location):
"""
Using the tar archive file at `location`, return an iterable of
name->value mappings for each tarfile.TarInfo objects essential
attributes.
Paths are base64-encoded because JSON is UTF-8 and cannot handle
arbitrary binary pathdata.
"""
with closing(tarfile.open(location)) as tar:
for entry in tar:
path = surrogate_decode(entry.path or '')
if entry.isdir() and not path.endswith('/'):
path += '/'
# libarchive introduces prefixes such as h prefix for
# hardlinks: tarfile does not, so we ignore the first char
mode = tarfile.filemode(entry.mode)[1:]
yield {
'path': path,
'mtime': entry.mtime,
'size': entry.size,
'mode': mode,
'isreg': entry.isreg(),
'isdir': entry.isdir(),
'islnk': entry.islnk(),
'issym': entry.issym(),
'linkpath': surrogate_decode(entry.linkpath or None),
'isblk': entry.isblk(),
'ischr': entry.ischr(),
'isfifo': entry.isfifo(),
'isdev': entry.isdev(),
}
def get_tarinfos(location):
"""
Using the tar archive file at `location`, return an iterable of
name->value mappings for each tarfile.TarInfo objects essential
attributes.
Paths are base64-encoded because JSON is UTF-8 and cannot handle
arbitrary binary pathdata.
"""
with closing(tarfile.open(location)) as tar:
for entry in tar:
path = surrogate_decode(entry.path or '')
if entry.isdir() and not path.endswith('/'):
path += '/'
# libarchive introduces prefixes such as h prefix for
# hardlinks: tarfile does not, so we ignore the first char
mode = tarfile.filemode(entry.mode)[1:]
yield {
'path': path,
'mtime': entry.mtime,
'size': entry.size,
'mode': mode,
'isreg': entry.isreg(),
'isdir': entry.isdir(),
'islnk': entry.islnk(),
'issym': entry.issym(),
'linkpath': surrogate_decode(entry.linkpath or None),
'isblk': entry.isblk(),
'ischr': entry.ischr(),
'isfifo': entry.isfifo(),
'isdev': entry.isdev(),
'uid': entry.uid,
'gid': entry.gid
}
def _stat_path(self, path):
hidden = False
base_name = path.split('/')[-1]
if base_name.startswith('.'):
hidden = True
stat_result = os.stat(self.filesystem.getsyspath(path))
try:
last_modified = time.strftime("%b %d %H:%M", time.localtime(stat_result.st_mtime))
except ValueError:
last_modified = time.strftime("%b %d %H:%M")
name = os.path.basename(path) or '.'
total = stat_result.st_blocks
if self.args.l:
path_string = "%s %2s %s %s %6s %s %s" % (
filemode(stat_result.st_mode),
stat_result.st_nlink,
'ftp',
'ftp',
stat_result.st_size,
last_modified,
name
)
else:
path_string = name
return {'path_string': path_string, 'total': total, 'hidden': hidden}
def get_list_dir(self, path):
path = self.fs2ftp(path)
print "******** listing files in %s" % path
node = self.__get_node(path)
for name in node.get_child_names():
stats = node.get_child_node(name).get_stats()
perms = tarfile.filemode(stats[stat.ST_MODE])
nlink = stats[stat.ST_NLINK]
if pwd and grp:
print "QUERYING PWD"
try:
user_name = pwd.getpwuid(stats[stat.ST_UID]).pw_name
group_name = grp.getgrgid(stats[stat.ST_GID]).gr_name
except:
traceback.print_exc()
user_name = str(stats[stat.ST_UID])
group_name = str(stats[stat.ST_GID])
else:
user_name = str(stats[stat.ST_UID])
group_name = str(stats[stat.ST_GID])
time_str = time.strftime("%b %d %H:%M",
time.localtime(stats[stat.ST_MTIME]))
yield "%s %3d %-8s %-8s %8d %s %s\r\n" % (
perms,
nlink,
user_name,
group_name,
stats[stat.ST_SIZE],
time_str,
name.encode('utf8'))
def format_filemode(filetype, filemode): if filetype == 2: prefix = "l" elif filetype == 5: prefix = "d" else: prefix = "-" return prefix + tarfile.filemode(filemode)[1:]
def create_filelist(self, datafile):
filelist = self.mktemp()
f = open(filelist, "w")
if self.payload_compression == "xz":
datafile = tar.InnerTarFileXz.open(datafile)
else:
datafile = tar.InnerTarFile.open(datafile)
while True:
m = datafile.next()
if not m:
break
log.debug(" %s %-8s %-8s %s %6s %s" % \
(tarfile.filemode(m.mode), m.uname, m.gname,
"%d-%02d-%02d %02d:%02d:%02d" % time.localtime(m.mtime)[:6],
util.format_size(m.size), m.name))
f.write("%(type)1s %(size)-10d %(uname)-10s %(gname)-10s %(mode)-6d %(mtime)-12d" \
% m.get_info(tarfile.ENCODING, "strict"))
# Calculate SHA512 hash of regular files.
if m.isreg():
mobj = datafile.extractfile(m)
h = hashlib.new("sha512")
while True:
buf = mobj.read(BUFFER_SIZE)
if not buf:
break
h.update(buf)
mobj.close()
f.write(" %s" % h.hexdigest())
else:
f.write(" -")
caps = m.pax_headers.get("PAKFIRE.capabilities", None)
if caps:
f.write(" %s" % caps)
else:
f.write(" -")
# The file name must be the last argument to contain spaces.
f.write(" %s" % m.name)
f.write("\n")
log.info("")
datafile.close()
f.close()
return filelist
def _stat_relative_dirs(self, path, name=None):
stat_result = os.stat(path)
try:
last_modified = time.strftime("%b %d %H:%M",
time.localtime(stat_result.st_mtime))
except ValueError:
last_modified = time.strftime("%b %d %H:%M")
return "%s %2s %s %s %6s %s %s" % (
filemode(stat_result.st_mode), stat_result.st_nlink, 'ftp', 'ftp',
stat_result.st_size, last_modified, name)
def _stat_relative_dirs(self, path, name=None):
stat_result = os.stat(path)
try:
last_modified = time.strftime("%b %d %H:%M", time.localtime(stat_result.st_mtime))
except ValueError:
last_modified = time.strftime("%b %d %H:%M")
return "%s %2s %s %s %6s %s %s" % (
filemode(stat_result.st_mode),
stat_result.st_nlink,
'ftp',
'ftp',
stat_result.st_size,
last_modified,
name
)
def format_list(self, cr, parent_node, listing, ignore_err=True):
"""Return an iterator object that yields the entries of given
directory emulating the "/bin/ls -lA" UNIX command output.
- (str) basedir: the parent directory node. Can be None
- (list) listing: a list of nodes
- (bool) ignore_err: when False raise exception if os.lstat()
call fails.
On platforms which do not support the pwd and grp modules (such
as Windows), ownership is printed as "owner" and "group" as a
default, and number of hard links is always "1". On UNIX
systems, the actual owner, group, and number of links are
printed.
This is how output appears to client:
-rw-rw-rw- 1 owner group 7045120 Sep 02 3:47 music.mp3
drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books
-rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py
"""
for node in listing:
perms = filemode(node.unixperms) # permissions
nlinks = 1
size = node.content_length or 0L
uname = _to_decode(node.uuser)
gname = _to_decode(node.ugroup)
# stat.st_mtime could fail (-1) if last mtime is too old
# in which case we return the local time as last mtime
try:
st_mtime = node.write_date or 0.0
if isinstance(st_mtime, basestring):
st_mtime = time.strptime(st_mtime, '%Y-%m-%d %H:%M:%S')
elif isinstance(st_mtime, float):
st_mtime = time.localtime(st_mtime)
mname=_get_month_name(time.strftime("%m", st_mtime ))
mtime = mname+' '+time.strftime("%d %H:%M", st_mtime)
except ValueError:
mname=_get_month_name(time.strftime("%m"))
mtime = mname+' '+time.strftime("%d %H:%M")
fpath = node.path
if isinstance(fpath, (list, tuple)):
fpath = fpath[-1]
# formatting is matched with proftpd ls output
path=_to_decode(fpath)
yield "%s %3s %-8s %-8s %8s %s %s\r\n" %(perms, nlinks, uname, gname,
size, mtime, path)
def format_list(self, cr, parent_node, listing, ignore_err=True):
"""Return an iterator object that yields the entries of given
directory emulating the "/bin/ls -lA" UNIX command output.
- (str) basedir: the parent directory node. Can be None
- (list) listing: a list of nodes
- (bool) ignore_err: when False raise exception if os.lstat()
call fails.
On platforms which do not support the pwd and grp modules (such
as Windows), ownership is printed as "owner" and "group" as a
default, and number of hard links is always "1". On UNIX
systems, the actual owner, group, and number of links are
printed.
This is how output appears to client:
-rw-rw-rw- 1 owner group 7045120 Sep 02 3:47 music.mp3
drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books
-rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py
"""
for node in listing:
perms = filemode(node.unixperms) # permissions
nlinks = 1
size = node.content_length or 0L
uname = _to_decode(node.uuser)
gname = _to_decode(node.ugroup)
# stat.st_mtime could fail (-1) if last mtime is too old
# in which case we return the local time as last mtime
try:
st_mtime = node.write_date or 0.0
if isinstance(st_mtime, basestring):
st_mtime = time.strptime(st_mtime, '%Y-%m-%d %H:%M:%S')
elif isinstance(st_mtime, float):
st_mtime = time.localtime(st_mtime)
mname = _get_month_name(time.strftime("%m", st_mtime))
mtime = mname + ' ' + time.strftime("%d %H:%M", st_mtime)
except ValueError:
mname = _get_month_name(time.strftime("%m"))
mtime = mname + ' ' + time.strftime("%d %H:%M")
fpath = node.path
if isinstance(fpath, (list, tuple)):
fpath = fpath[-1]
# formatting is matched with proftpd ls output
path = _to_decode(fpath)
yield "%s %3s %-8s %-8s %8s %s %s\r\n" % (perms, nlinks, uname,
gname, size, mtime, path)
def format_list(self, basedir, listing, ignore_err=True):
"""Return an iterator object that yields the entries of given
directory emulating the "/bin/ls -lA" UNIX command output.
- (str) basedir: the absolute dirname.
- (list) listing: the names of the entries in basedir
- (bool) ignore_err: when False raise exception if os.lstat()
call fails.
On platforms which do not support the pwd and grp modules (such
as Windows), ownership is printed as "owner" and "group" as a
default, and number of hard links is always "1". On UNIX
systems, the actual owner, group, and number of links are
printed.
This is how output appears to client:
-rw-rw-rw- 1 owner group 7045120 Sep 02 3:47 music.mp3
drwxrwxrwx 1 owner group 0 Aug 31 18:50 e-books
-rw-rw-rw- 1 owner group 380 Sep 02 3:40 module.py
"""
for file in listing:
try:
st = self.lstat(file)
except os.error:
if ignore_err:
continue
raise
perms = filemode(st.st_mode) # permissions
nlinks = st.st_nlink # number of links to inode
if not nlinks: # non-posix system, let's use a bogus value
nlinks = 1
size = st.st_size # file size
uname = "owner"
gname = "group"
# stat.st_mtime could fail (-1) if last mtime is too old
# in which case we return the local time as last mtime
try:
mname=_get_month_name(time.strftime("%m", time.localtime(st.st_mtime)))
mtime = mname+' '+time.strftime("%d %H:%M", time.localtime(st.st_mtime))
except ValueError:
mname=_get_month_name(time.strftime("%m"))
mtime = mname+' '+time.strftime("%d %H:%M")
# formatting is matched with proftpd ls output
path=_to_decode(file.path) #file.path.encode('ascii','replace').replace('?','_')
yield "%s %3s %-8s %-8s %8s %s %s\r\n" %(perms, nlinks, uname, gname,
size, mtime, path.split('/')[-1])
def _convert_dacs(self, dacs):
dacval = 0
if dacs == None:
raise KeyError("_convert_dacs -> no DAC data given to translate")
elif type(dacs) == type('string'):
dacval = int(dacs, 8)
elif type(dacs) == type(0):
dacval = dacs
else :
raise KeyError ("_convert_dacs -> invalid data given ")
dacval &= 07777 # strip off 'type' info, keeping values suitable for chmod
dacs_map = {}
dacs_map['dacs_value'] = dacval
dacs_map['dacs_str'] = oct(dacval)
dacs_map['dacs_flags'] = tarfile.filemode(dacs_map['dacs_value'])
return dacs_map
def list(self):
for member in self.members:
output=""
output= tarfile.filemode(member.mode)
output= output +( " %s/%s " % (member.uname or member.uid, member.gname or member.gid))
if member.ischr() or member.isblk():
output = output+( "%10s " % ("%d,%d" % (member.devmajor, member.devminor)) )
else:
output = output+ "%10d" % member.size
output = output + " %d-%02d-%02d %02d:%02d:%02d " % time.localtime(member.mtime)[:6]
output = output + member.name + ("/" if member.isdir() else "")
if member.issym():
output=output + "->", member.linkname,
if member.islnk():
output = output + "link to", member.linkname,
self.debug(output)
def list(self):
for member in self.members:
output = ""
output = tarfile.filemode(member.mode)
output = output + (" %s/%s " % (member.uname or member.uid,
member.gname or member.gid))
if member.ischr() or member.isblk():
output = output + ("%10s " % ("%d,%d" % (member.devmajor, member.devminor)))
else:
output = output + "%10d" % member.size
output = output + " %d-%02d-%02d %02d:%02d:%02d " % time.localtime(member.mtime)[:6]
output = output + member.name + ("/" if member.isdir() else "")
if member.issym():
output = output + "->", member.linkname,
if member.islnk():
output = output + "link to", member.linkname,
self.debug(output)
def _stat_path(self, path):
hidden = False
base_name = path.split('/')[-1]
if base_name.startswith('.'):
hidden = True
stat_result = os.stat(self.filesystem.getsyspath(path))
try:
last_modified = time.strftime("%b %d %H:%M",
time.localtime(stat_result.st_mtime))
except ValueError:
last_modified = time.strftime("%b %d %H:%M")
name = os.path.basename(path) or '.'
total = stat_result.st_blocks
if self.args.l:
path_string = "%s %2s %s %s %6s %s %s" % (
filemode(stat_result.st_mode), stat_result.st_nlink, 'ftp',
'ftp', stat_result.st_size, last_modified, name)
else:
path_string = name
return {'path_string': path_string, 'total': total, 'hidden': hidden}
def list_contents(tar_file):
"""
Listing contents from a tar file. This implementation is based on
Python core code, (tarfile.list).
"""
members = tar_file.getmembers()
output = ''
for tarinfo in members:
line = ''
line += tarfile.filemode(tarinfo.mode) + ' '
owner = '{0}/{1}'.format(
tarinfo.uname or tarinfo.uid,
tarinfo.gname or tarinfo.gid
)
line += owner
if tarinfo.ischr() or tarinfo.isblk():
dev = '{0},{1}'.format(tarinfo.devmajor, tarinfo.devminor)
line += dev.center(10) + ' '
else:
line += '{0:10}'.format(tarinfo.size) + ' '
line += str(datetime.fromtimestamp(tarinfo.mtime)) + ' '
line += '{0}{1}{2}'.format(
tarinfo.name if PY3 else tarinfo.name.decode('utf-8'),
('/' if tarinfo.isdir() else ''),
' '
)
if tarinfo.issym():
line += '-> {0}'.format(tarinfo.linkname)
if tarinfo.islnk():
line += 'link to {0}'.format(tarinfo.linkname)
output += '{0}\n'.format(line)
return output or 'Nothing to output.'
def expand_dirent(entry):
''' Utility function to interpret the json values as python where approprate '''
# these should be int() converted
convert_to_int = set(
('size', 'block_size', 'mtime_val', 'atime_val', 'ctime_val',
'btime_val', 'uid', 'gid', 'id', 'nlink'))
# there should be fromtimestamp() converted
convert_to_time = set(
('mtime_val', 'atime_val', 'ctime_val', 'btime_val'))
# this is the mapping of type to mode
type_to_flag = {
'container': 0o0040000,
'object': 0o0100000,
'pipe': 0o0010000,
'character_device': 0o0020000,
'block_device': 0o0060000,
'symbolic_link': 0o0120000,
'socket': 0o0140000,
'whiteout_file': 0
}
for key in convert_to_int:
if key in entry:
entry[key] = int(entry[key])
for key in convert_to_time:
if key in entry:
entry[key] = datetime.fromtimestamp(entry[key])
if 'size' in entry:
entry['hsize'] = sfmt(entry['size'])
if 'mode' in entry:
entry['mode'] = int(entry['mode'], 8)
try:
entry['mode'] = entry['mode'] | type_to_flag[entry['type']]
except BaseException:
pass
entry['mode_str'] = filemode(entry['mode'])
def test_modes(self):
self.assertEqual(tarfile.filemode(0755), '-rwxr-xr-x')
self.assertEqual(tarfile.filemode(07111), '---s--s--t')
def test_modes(self):
self.assertEqual(tarfile.filemode(0755), "-rwxr-xr-x")
self.assertEqual(tarfile.filemode(07111), "---s--s--t")
while True:
try:
file = snapshot.next()
except EOFError:
break
if file.isref():
if self.opts.list_new:
continue
else:
file = snapshot.dereference(file)
if includer.match(
file.name) and not self.excluder.match(file.name):
if self.opts.verbose:
print tarfile.filemode(file.mode),
print text_encode(file.uname), text_encode(file.gname),
print "%10d" % file.size,
print "%d-%02d-%02d %02d:%02d" % time.localtime(
file.mtime)[:5],
print text_encode(file.name),
if self.opts.verbose:
if file.issym():
print "->", text_encode(file.link),
print
def extract(self):
"""Extract files from a backup snapshot.
"""
