def test_corrupted_extract_selective(self): path1 = self.corruptedarchivepath checked = DirectoryTree(self.corruptedarchivepath, self.testcontainer) checked.add(path1, True) tmpdir = tempfile.mkdtemp() LibArchive.extract(self.testcontainer, container(self.corruptedarchivepath).archive, tmpdir, checked) n = 0 for prefix, files, dirs in os.walk(tmpdir): n += len(files) + len(dirs) self.assertEqual(n, 0) # clean up for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir)
def test_add_file(self): tree = DirectoryTree(self.testdatapath, self.fs) path1 = self.fs.join(self.testdatapath, 'a', 'aa', 'aaa') path2 = self.fs.join(self.testdatapath, 'a', 'aa') tree.add(path1) self.assertIn(path1, tree) self.assertIn(path2, tree) # not-added one-level-up directory
def test_create_utf8(self): tmpdir = tempfile.mkdtemp() testarchivepath = os.path.join(self.testdirectory, 'testdata', 'tešt.tar') LibArchive.extract(self.testcontainer, container(testarchivepath).archive, tmpdir) path1 = os.path.join(tmpdir, 'tarmanš.log') fs = FileSystem() checked = DirectoryTree(tmpdir, fs) checked.add(path1, False) testdata2archivepath = os.path.join(tmpdir, 'tešt2.tar') LibArchive.create(fs, testdata2archivepath, checked) self.assertTrue(os.path.exists(testdata2archivepath)) testdata2archive = LibArchive(testdata2archivepath) apath1 = os.path.join(testdata2archivepath, 'tarmanš.log') self.assertIn(apath1, testdata2archive.tree) # clean up for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir)
def test_verify_with_checked(self): checked = DirectoryTree( self.testarchivepath, self.testcontainer ) checked.add(self.testcontainer.join(self.testarchivepath, 'a'), True) self.assertIn( self.testcontainer.join(self.testarchivepath, 'a', 'aa'), checked ) self.assertFalse( LibArchive.verify( self.testarchivepath, 'abc', checked ) ) self.assertTrue( LibArchive.verify( self.testarchivepath, 'a/aa/aaa', checked ) ) self.assertTrue( LibArchive.verify( self.testarchivepath, 'a/ab', checked ) )
def test_create(self): testdatadirectory = os.path.join(self.testdirectory, 'testdata', 'testdata') path1 = os.path.join(testdatadirectory, 'a', 'aa', 'aaa') path2 = os.path.join(testdatadirectory, 'c') path3 = os.path.join(testdatadirectory, 'a', 'ab') fs = FileSystem() checked = DirectoryTree(testdatadirectory, fs) checked.add(path1, False) checked.add(path2, False) checked.add(path3, True) tmpdir = tempfile.mkdtemp() testdata2archivepath = os.path.join(tmpdir, 'testdata2.tar.gz') LibArchive.create(fs, testdata2archivepath, checked) self.assertTrue(os.path.exists(testdata2archivepath)) testdata2archive = LibArchive(testdata2archivepath) apath1 = os.path.join(testdata2archivepath, 'a', 'aa', 'aaa') apath2 = os.path.join(testdata2archivepath, 'c') apath3 = os.path.join(testdata2archivepath, 'a', 'ab') self.assertIn(apath1, testdata2archive.tree) self.assertIn(apath2, testdata2archive.tree) self.assertIn(apath3, testdata2archive.tree) os.remove(testdata2archivepath) os.rmdir(tmpdir)
def test_corrupted_extract_selective(self): path1 = self.corruptedarchivepath checked = DirectoryTree( self.corruptedarchivepath, self.testcontainer ) checked.add(path1, True) tmpdir = tempfile.mkdtemp() LibArchive.extract( self.testcontainer, container(self.corruptedarchivepath).archive, tmpdir, checked ) n = 0 for prefix, files, dirs in os.walk(tmpdir): n += len(files) + len(dirs) self.assertEqual(n, 0) # clean up for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir)
def test_corrupted_create(self): tmpdir = tempfile.mkdtemp() LibArchive.extract(self.testcontainer, container(self.corruptedarchivepath).archive, tmpdir) fs = FileSystem() path1 = tmpdir checked = DirectoryTree(tmpdir, fs) checked.add(path1, True) corrupted2archivepath = os.path.join(tmpdir, 'corrupted2.tar') LibArchive.create(fs, corrupted2archivepath, checked) self.assertTrue(os.path.exists(corrupted2archivepath)) corrupted2archive = LibArchive(corrupted2archivepath) self.assertEqual(len(corrupted2archive.tree.root.children), 0) # clean up for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir)
def test_out_of_range(self): tree = DirectoryTree(self.testdatapath, self.fs) # one level up directory dir1 = self.fs.abspath(self.fs.join(self.testdatapath, '..')) with self.assertRaises(OutOfRange): tree.add(dir1) # completely different directory with tempfile.NamedTemporaryFile() as f: with self.assertRaises(OutOfRange): tree.add(f.name)
def test_verify_with_checked(self): checked = DirectoryTree(self.testarchivepath, self.testcontainer) checked.add(self.testcontainer.join(self.testarchivepath, 'a'), True) self.assertIn(self.testcontainer.join(self.testarchivepath, 'a', 'aa'), checked) self.assertFalse( LibArchive.verify(self.testarchivepath, 'abc', checked)) self.assertTrue( LibArchive.verify(self.testarchivepath, 'a/aa/aaa', checked)) self.assertTrue( LibArchive.verify(self.testarchivepath, 'a/ab', checked))
def test_create_utf8(self): tmpdir = tempfile.mkdtemp() testarchivepath = os.path.join( self.testdirectory, 'testdata', 'tešt.tar' ) LibArchive.extract( self.testcontainer, container(testarchivepath).archive, tmpdir ) path1 = os.path.join(tmpdir, 'tarmanš.log') fs = FileSystem() checked = DirectoryTree( tmpdir, fs ) checked.add(path1, False) testdata2archivepath = os.path.join( tmpdir, 'tešt2.tar' ) LibArchive.create( fs, testdata2archivepath, checked ) self.assertTrue(os.path.exists(testdata2archivepath)) testdata2archive = LibArchive(testdata2archivepath) apath1 = os.path.join(testdata2archivepath, 'tarmanš.log') self.assertIn(apath1, testdata2archive.tree) # clean up for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir)
class Zip(Container, Archive): def __init__(self, path): self.path = os.path.abspath(path) self.archive = Zip.open(self.path) self.tree = DirectoryTree(self.path, self) names = self.archive.namelist() for n in names: if n[-1] == os.sep: continue self.tree.add(os.path.join(self.path, n)) def listdir(self, path): children = self.tree[path].children return [c.data for c in children] def isenterable(self, path): children = self.tree[path].children return True if children else False def abspath(self, path): return self.tree[path].get_path() @staticmethod def isarchive(path): return False # disable return zipfile.is_zipfile(path) @staticmethod def open(path): return zipfile.ZipFile(file=path) @staticmethod def extract(container, archive, target_path, checked=None): if checked: members = [] for node in checked: # without root data arr = container.tree[node.get_path()].get_data_array()[1:] if arr[0] == '..': continue members += [archive.getinfo(os.sep.join(arr))] else: members = None archive.extractall(path=target_path, members=members)
class Tar(Container, Archive): def __init__(self, path): self.path = os.path.abspath(path) self.archive = Tar.open(self.path) self.tree = DirectoryTree(self.path, self) names = self.archive.getnames() for n in names: self.tree.add(os.path.join(self.path, n)) def listdir(self, path): children = self.tree[path].children return [c.data for c in children] def isenterable(self, path): children = self.tree[path].children return True if children else False def abspath(self, path): return self.tree[path].get_path() @staticmethod def isarchive(path): return tarfile.is_tarfile(path) @staticmethod def open(path): return tarfile.open(path) @staticmethod def extract(archive, target_path, checked=None): if checked: members = [] for node in checked: arr = node.get_data_array()[1:] # without root data if arr[0] == '..': continue members += [archive.getmember(os.sep.join(arr))] else: members = None archive.extractall(path=target_path, members=members)
def test_corrupted_create(self): tmpdir = tempfile.mkdtemp() LibArchive.extract( self.testcontainer, container(self.corruptedarchivepath).archive, tmpdir ) fs = FileSystem() path1 = tmpdir checked = DirectoryTree( tmpdir, fs ) checked.add(path1, True) corrupted2archivepath = os.path.join( tmpdir, 'corrupted2.tar' ) LibArchive.create( fs, corrupted2archivepath, checked ) self.assertTrue(os.path.exists(corrupted2archivepath)) corrupted2archive = LibArchive(corrupted2archivepath) self.assertEqual(len(corrupted2archive.tree.root.children), 0) # clean up for root, dirs, files in os.walk(tmpdir, topdown=False): for name in files: os.remove(os.path.join(root, name)) for name in dirs: os.rmdir(os.path.join(root, name)) os.rmdir(tmpdir)
def test_create(self): testdatadirectory = os.path.join( self.testdirectory, 'testdata', 'testdata' ) path1 = os.path.join(testdatadirectory, 'a', 'aa', 'aaa') path2 = os.path.join(testdatadirectory, 'c') path3 = os.path.join(testdatadirectory, 'a', 'ab') fs = FileSystem() checked = DirectoryTree( testdatadirectory, fs ) checked.add(path1, False) checked.add(path2, False) checked.add(path3, True) tmpdir = tempfile.mkdtemp() testdata2archivepath = os.path.join( tmpdir, 'testdata2.tar.gz' ) LibArchive.create( fs, testdata2archivepath, checked ) self.assertTrue(os.path.exists(testdata2archivepath)) testdata2archive = LibArchive(testdata2archivepath) apath1 = os.path.join(testdata2archivepath, 'a', 'aa', 'aaa') apath2 = os.path.join(testdata2archivepath, 'c') apath3 = os.path.join(testdata2archivepath, 'a', 'ab') self.assertIn(apath1, testdata2archive.tree) self.assertIn(apath2, testdata2archive.tree) self.assertIn(apath3, testdata2archive.tree) os.remove(testdata2archivepath) os.rmdir(tmpdir)
class Main(object): @utf8_args(3) def __init__(self, mainscr, stdscr, directory, encoding): self.encoding = encoding self.header_lns = HEADER_LNS self.mainscr = mainscr self.stdscr = stdscr self.color = curses.has_colors() if self.color: # set file type attributes (color and bold) curses.init_pair(1, curses.COLOR_BLUE, -1) self.attr_folder = curses.color_pair(1) | curses.A_BOLD curses.init_pair(2, 7, -1) self.attr_norm = curses.color_pair(2) # set wright / wrong attributes (color and bold) curses.init_pair(3, curses.COLOR_GREEN, -1) self.attr_wright = curses.color_pair(3) | curses.A_BOLD curses.init_pair(4, curses.COLOR_RED, -1) self.attr_wrong = curses.color_pair(4) | curses.A_BOLD self.kill = False self.ch = -1 self.visited = {} self.area = None self.container = FileSystem() self.directory = self.container.abspath(directory) self.checked = DirectoryTree(self.directory, self.container) self.chdir(self.directory) @utf8_args(1, 2) def header(self, prefix, path): #self.mainscr.clear() h, w = self.mainscr.getmaxyx() sep = " " length = len(prefix) + len(path) + len(sep) empty = 0 if length > w: path = "..." + path[length - w + 3:] else: empty = w - length self.mainscr.addstr( 0, 0, u"{0}{1}{2}{3}".format( prefix, sep, path, empty * ' ' ).encode(self.encoding) ) self.mainscr.refresh() def identify_container_and_checked(self, path): if self.container.isenterable(path): # is folder return self.container, self.checked # force one-level archive browsing if not isinstance(self.container, FileSystem): return None, None aclass = get_archive_class(path) if not aclass: return None, None workwin = WorkWin(self) workwin.show("Working ...") newcontainer = aclass(path) newchecked = DirectoryTree(path, newcontainer) workwin.close() return newcontainer, newchecked def chdir(self, newpath): if newpath is None: return False if not newpath.startswith(self.directory): return False try: if self.area is None: oldsel = 0 oldpath = self.directory else: oldsel = self.area.selected oldpath = self.area.abspath oldcontainer = self.container oldchecked = self.checked if newpath in self.visited: newsel, newcontainer, newchecked = self.visited[newpath] else: newcontainer, newchecked = \ self.identify_container_and_checked(newpath) if newcontainer is None: return False newsel = 0 self.visited[oldpath] = [oldsel, oldcontainer, oldchecked] logging.info(u"OLD - {0} - {1} - {2}".format( oldpath, oldsel, oldcontainer.__class__.__name__ )) logging.info(u"NEW - {0} - {1} - {2}".format( newpath, newsel, newcontainer.__class__.__name__ )) h, w = self.stdscr.getmaxyx() self.container = newcontainer self.checked = newchecked def show_unreadable_error(path, name): if isinstance(name, unicode): error_str = name.encode('utf8', errors='replace') else: error_str = name logging.info("Unreadable file name: {0} (in '{1}')".format( error_str, path )) errorwin = TextWin(self) errorwin.show("Unreadable file name:\n{0}\n\nPress ESC to close.".format(error_str)) self.area = ViewArea( newpath, h, newcontainer, show_unreadable_error ) self.header( "{0}".format( self.container.__class__.__name__ ), self.area.abspath ) self.area.set_params(h, offset=newsel) self.refresh_scr() return True except OutOfRange: logging.error("OutOfRange .. {0}".format(newpath)) curses.flash() def insert_line(self, y, item): i, name, abspath = item self.stdscr.addstr( y, 0, "[{0}]".format( '*' if abspath in self.checked else ' ' ).encode(self.encoding) ) if self.color: if self.container.isenterable(abspath): attr = self.attr_folder name = u"{0}/".format(name) else: attr = self.attr_norm self.stdscr.addstr(y, 5, name.encode(self.encoding), attr) else: if self.container.isenterable(abspath): name = u"{0}/".format(name) self.stdscr.addstr(y, 5, name.encode(self.encoding)) def refresh_scr(self): self.stdscr.clear() if not getattr(self, 'area', None): return if len(self.area) == 0: self.stdscr.addstr(1, 5, "Directory is empty!") return iitem = 0 for item in self.area: self.insert_line(iitem, item) iitem += 1 y = self.area.selected_local h, w = self.stdscr.getmaxyx() self.stdscr.chgat(y, 0, w, curses.A_REVERSE) self.stdscr.move(y, 1) def loop(self): while not self.kill: self.ch = self.stdscr.getch() h, w = self.stdscr.getmaxyx() if self.ch in [ord('q'), ]: self.kill = True elif self.ch == curses.KEY_UP: self.area.set_params(h, offset=-1) elif self.ch == curses.KEY_DOWN: self.area.set_params(h, offset=1) elif self.ch == curses.KEY_PPAGE: self.area.set_params(h, offset=-5) elif self.ch == curses.KEY_NPAGE: self.area.set_params(h, offset=5) elif self.ch == 32: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if abspath in self.checked: del self.checked[abspath] else: countitems = self.container.count_items( abspath, stop_at=ITEMS_WARNING ) if countitems >= ITEMS_WARNING and \ self.show_items_warning() != 0: continue self.checked.add(abspath, sub=True) elif self.ch in [curses.KEY_RIGHT, 10, 13]: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) result = self.chdir(abspath) if not result: curses.flash() elif self.ch in [curses.KEY_LEFT, 127, curses.ascii.BS, curses.KEY_BACKSPACE]: if not self.chdir( self.container.dirname(self.area.abspath) ): curses.flash() elif self.ch in [ord('c'), ord('C')]: if isinstance(self.container, FileSystem): aclass = self.container.__class__ checked = self.checked container = self.container pathwin = PathWin(self) exitstatus, archivepath = pathwin.show( "Create archive with format/compression based on file" " extension (ENTER to confirm or ESC to cancel):", os.path.join(os.getcwd(), "NewArchive.tar.gz") ) pathwin.close() logging.info("window exitstatus: {0}, '{1}'".format( exitstatus, archivepath )) if exitstatus != 0: continue archivepath = os.path.abspath(archivepath) aclass = get_archive_class(archivepath) if aclass is None: curses.flash() continue created = aclass.create(container, archivepath, checked) if created: TextWin(self).show( "Successfully created archive:\n{0}".format( archivepath ) ) else: curses.flash() elif self.ch in [ord('e'), ord('E')]: if isinstance(self.container, Archive): aclass = self.container.__class__ archive = self.container.archive checked = self.checked container = self.container else: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if not abspath: curses.flash() continue aclass = get_archive_class(abspath) if aclass is None: curses.flash() continue archive = aclass.open(abspath) checked = None container = None pathwin = PathWin(self) exitstatus, s = pathwin.show( "Extract to " "(press ENTER for confirmation or ESC to cancel):" ) pathwin.close() logging.info("window exitstatus: {0}, '{1}'".format( exitstatus, s )) if exitstatus != 0: continue workwin = WorkWin(self) workwin.show("Extracting ...") aclass.extract(container, archive, s, checked=checked) workwin.close() TextWin(self).show("Extracted to:\n{0}".format(s)) elif self.ch in [ord('?'), curses.KEY_F1, ord('h')]: textwin = TextWin(self) textwin.show(HELP_STRING) if self.ch != -1: self.refresh_scr() if self.kill: break def show_items_warning(self): questionwin = QuestionWin(self) return questionwin.show( "There are more than {0} items in this folder," "\ndo you really want to select it?".format( ITEMS_WARNING ) ) def cancel(self): self.kill = True
class LibArchive(Container, Archive): def __init__(self, path): self.path = s2u(os.path.abspath(path)) self.tree = DirectoryTree(self.path, self) with LibArchive.open(self.path) as larchive: self.archive = larchive a = larchive._a while True: try: e = _libarchive.archive_entry_new() r = _libarchive.archive_read_next_header2(a, e) if r != _libarchive.ARCHIVE_OK: break name = _libarchive.archive_entry_pathname(e) pathname = s2u(name) if pathname[-1] == '/': pathname = pathname[:-1] self.tree.add(os.path.join(self.path, pathname)) except UnicodeDecodeError: logging.info("Unreadable file name: {0} (in '{1}')".format( name, self.path)) finally: _libarchive.archive_entry_free(e) def listdir(self, path): children = self.tree[path].children return [c.data for c in children] def isenterable(self, path): children = self.tree[path].children return True if children else False def abspath(self, path): return self.tree[path].get_path() def count_items(self, path, stop_at=-1): node = self.tree[path] n = 0 for p in node.__iter__(): n += 1 if n == stop_at: break return n @staticmethod def isarchive(path): try: return libarchive.is_archive(path) except: return libarchive.is_archive_name(path) @staticmethod def open(path): return libarchive.Archive(u2s(path)) @staticmethod @utf8_args(0, 1) def verify(archive_path, pathname, checked): # do not allow to go up in the directory tree like that if pathname.startswith('..'): return False # do not allow absolute paths if pathname[0] == '/': return False # if none is checked, that means, extract all if not checked: return True # if file is checked, extract it try: if "/".join([archive_path, pathname]) in checked: return True except OutOfRange: # this is expected return False # return false if file is not checked return False @staticmethod @utf8_args(2) def extract(container, archive, target_path, checked=None): target_path = os.path.abspath(target_path) archive_path = s2u(archive.filename) # reopen archive file archive.denit() archive.f.seek(0) archive.init() a = archive._a try: if checked: logging.info( u"START extract selective '{0}'".format(archive_path)) while True: try: e = _libarchive.archive_entry_new() r = _libarchive.archive_read_next_header2(a, e) if r != _libarchive.ARCHIVE_OK: break pathname = _libarchive.archive_entry_pathname(e) if pathname[-1] == '/': pathname = pathname[:-1] path = os.path.join(target_path, s2u(pathname)) logging.info(u"from '{0}' to '{1}'".format( s2u(pathname), s2u(path))) if LibArchive.verify(archive_path, pathname, checked): ftype = _libarchive.archive_entry_filetype(e) if stat.S_ISDIR(ftype): makepath(path) else: makepath(os.path.dirname(path)) with open(path, 'wb') as f: _libarchive.archive_read_data_into_fd( a, f.fileno()) finally: _libarchive.archive_entry_free(e) logging.info( u"END extract selective '{0}'".format(archive_path)) else: logging.info(u"START extract all '{0}'".format(archive_path)) while True: try: e = _libarchive.archive_entry_new() r = _libarchive.archive_read_next_header2(a, e) if r != _libarchive.ARCHIVE_OK: break pathname = _libarchive.archive_entry_pathname(e) path = os.path.join(target_path, s2u(pathname)) logging.info(u"from '{0}' to '{1}'".format( s2u(pathname), s2u(path))) if stat.S_ISDIR(_libarchive.archive_entry_filetype(e)): makepath(path) else: makepath(os.path.dirname(path)) with open(u2s(path), 'wb') as f: _libarchive.archive_read_data_into_fd( a, f.fileno()) finally: _libarchive.archive_entry_free(e) logging.info(u"END extract all '{0}'".format(archive_path)) finally: archive.close() @staticmethod @utf8_args(1) def create2(container, archivepath, checked): if not isinstance(container, FileSystem): logging.info("container '{0}' is not FileSystem".format(container)) return False archivepath = os.path.abspath(archivepath) with libarchive.Archive(archivepath, 'w') as a: logging.info(u"START create selective '{0}'".format(archivepath)) for node in checked: path = s2u(node.get_path()) pathname = s2u(os.path.join(*node.get_data_array()[1:])) logging.info(u"from '{0}' to '{1}'".format(path, pathname)) a.writepath(path, pathname) logging.info(u"END create selective '{0}'".format(archivepath)) return True @staticmethod def create(container, archivepath, checked): if not isinstance(container, FileSystem): logging.info("container '{0}' is not FileSystem".format(container)) return False archivepath = os.path.abspath(u2s(archivepath)) BUFFER_SIZE = 2048 with libarchive.Archive(archivepath, 'w') as larchive: a = larchive._a for node in checked: if node.is_dir(): continue path = u2s(node.get_path()) pathname = u2s(os.path.join(*node.get_data_array()[1:])) st = os.stat(path) entry = _libarchive.archive_entry_new() _libarchive.archive_entry_set_pathname(entry, pathname) _libarchive.archive_entry_set_size(entry, st.st_size) _libarchive.archive_entry_set_filetype(entry, stat.S_IFMT(st.st_mode)) _libarchive.archive_entry_set_mtime(entry, st.st_mtime, 0) _libarchive.archive_entry_set_perm(entry, stat.S_IMODE(st.st_mode)) _libarchive.archive_write_header(a, entry) f = open(path, 'r') data = f.read(BUFFER_SIZE) count = len(data) while count > 0: _libarchive.archive_write_data_from_str(a, data) data = f.read(BUFFER_SIZE) count = len(data) f.close() _libarchive.archive_entry_free(entry) _libarchive.archive_write_close(a)
class Main(object): @utf8_args(3) def __init__(self, mainscr, stdscr, directory, encoding, show_hiddens): self.encoding = encoding self.header_lns = HEADER_LNS self.mainscr = mainscr self.stdscr = stdscr self.color = curses.has_colors() if self.color: # set file type attributes (color and bold) curses.init_pair(1, curses.COLOR_BLUE, -1) self.attr_folder = curses.color_pair(1) | curses.A_BOLD curses.init_pair(2, 7, -1) self.attr_norm = curses.color_pair(2) # set wright / wrong attributes (color and bold) curses.init_pair(3, curses.COLOR_GREEN, -1) self.attr_wright = curses.color_pair(3) | curses.A_BOLD curses.init_pair(4, curses.COLOR_RED, -1) self.attr_wrong = curses.color_pair(4) | curses.A_BOLD self.kill = False self.ch = -1 self.visited = {} self.area = None self.container = FileSystem() self.directory = self.container.abspath(directory) self.checked = DirectoryTree(self.directory, self.container) self.show_hiddens = show_hiddens self.chdir(self.directory) @utf8_args(1, 2) def header(self, prefix, path): #self.mainscr.clear() h, w = self.mainscr.getmaxyx() sep = " " length = len(prefix) + len(path) + len(sep) empty = 0 if length > w: path = "..." + path[length - w + 3:] else: empty = w - length self.mainscr.addstr( 0, 0, u"{0}{1}{2}{3}".format( prefix, sep, path, empty * ' ' ).encode(self.encoding) ) self.mainscr.refresh() def identify_container_and_checked(self, path): if self.container.isenterable(path): # is folder return self.container, self.checked # force one-level archive browsing if not isinstance(self.container, FileSystem): return None, None aclass = get_archive_class(path) if not aclass: return None, None workwin = WorkWin(self) workwin.show("Working ...") newcontainer = aclass(path) newchecked = DirectoryTree(path, newcontainer) workwin.close() return newcontainer, newchecked def chdir(self, newpath): if newpath is None: return False if not newpath.startswith(self.directory): return False try: if self.area is None: oldsel = 0 oldpath = self.directory else: oldsel = self.area.selected oldpath = self.area.abspath oldcontainer = self.container oldchecked = self.checked if newpath in self.visited: newsel, newcontainer, newchecked = self.visited[newpath] else: newcontainer, newchecked = \ self.identify_container_and_checked(newpath) if newcontainer is None: return False newsel = 0 self.visited[oldpath] = [oldsel, oldcontainer, oldchecked] logging.info(u"OLD - {0} - {1} - {2}".format( oldpath, oldsel, oldcontainer.__class__.__name__ )) logging.info(u"NEW - {0} - {1} - {2}".format( newpath, newsel, newcontainer.__class__.__name__ )) h, w = self.stdscr.getmaxyx() self.container = newcontainer self.checked = newchecked def show_unreadable_error(path, name): if isinstance(name, unicode): error_str = name.encode('utf8', errors='replace') else: error_str = name logging.info("Unreadable file name: {0} (in '{1}')".format( error_str, path )) errorwin = TextWin(self) errorwin.show("Unreadable file name:\n{0}\n\nPress ESC to close.".format(error_str)) self.area = ViewArea( newpath, h, newcontainer, show_unreadable_error, self.show_hiddens ) self.header( "{0}".format( self.container.__class__.__name__ ), self.area.abspath ) self.area.set_params(h, offset=newsel) self.refresh_scr() return True except OutOfRange: logging.error("OutOfRange .. {0}".format(newpath)) curses.flash() def insert_line(self, y, item): i, name, abspath = item self.stdscr.addstr( y, 0, "[{0}]".format( '*' if abspath in self.checked else ' ' ).encode(self.encoding) ) if self.color: if self.container.isenterable(abspath): attr = self.attr_folder name = u"{0}/".format(name) else: attr = self.attr_norm self.stdscr.addstr(y, 5, name.encode(self.encoding), attr) else: if self.container.isenterable(abspath): name = u"{0}/".format(name) self.stdscr.addstr(y, 5, name.encode(self.encoding)) def refresh_scr(self): self.stdscr.clear() if not getattr(self, 'area', None): return if len(self.area) == 0: self.stdscr.addstr(1, 5, "Directory is empty!") return iitem = 0 for item in self.area: self.insert_line(iitem, item) iitem += 1 y = self.area.selected_local h, w = self.stdscr.getmaxyx() self.stdscr.chgat(y, 0, w, curses.A_REVERSE) self.stdscr.move(y, 1) def loop(self): while not self.kill: self.ch = self.stdscr.getch() h, w = self.stdscr.getmaxyx() if self.ch in [ord('q'), ]: self.kill = True elif self.ch == curses.KEY_UP: self.area.set_params(h, offset=-1) elif self.ch == curses.KEY_DOWN: self.area.set_params(h, offset=1) elif self.ch == curses.KEY_PPAGE: self.area.set_params(h, offset=-5) elif self.ch == curses.KEY_NPAGE: self.area.set_params(h, offset=5) elif self.ch == 32: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if abspath in self.checked: del self.checked[abspath] else: countitems = self.container.count_items( abspath, stop_at=ITEMS_WARNING ) if countitems >= ITEMS_WARNING and \ self.show_items_warning() != 0: continue self.checked.add(abspath, sub=True) elif self.ch in [curses.KEY_RIGHT, 10, 13]: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) result = self.chdir(abspath) if not result: curses.flash() elif self.ch in [curses.KEY_LEFT, 127, curses.ascii.BS, curses.KEY_BACKSPACE]: if not self.chdir( self.container.dirname(self.area.abspath) ): curses.flash() elif self.ch in [ord('c'), ord('C')]: if isinstance(self.container, FileSystem): aclass = self.container.__class__ checked = self.checked container = self.container pathwin = PathWin(self) exitstatus, archivepath = pathwin.show( "Create archive with format/compression based on file" " extension (ENTER to confirm or ESC to cancel):", os.path.join(os.getcwd(), "NewArchive.tar.gz") ) pathwin.close() logging.info("window exitstatus: {0}, '{1}'".format( exitstatus, archivepath )) if exitstatus != 0: continue archivepath = os.path.abspath(archivepath) aclass = get_archive_class(archivepath) if aclass is None: curses.flash() continue created = aclass.create(container, archivepath, checked) if created: TextWin(self).show( "Successfully created archive:\n{0}".format( archivepath ) ) else: curses.flash() elif self.ch in [ord('e'), ord('E')]: if isinstance(self.container, Archive): aclass = self.container.__class__ archive = self.container.archive checked = self.checked container = self.container else: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if not abspath: curses.flash() continue aclass = get_archive_class(abspath) if aclass is None: curses.flash() continue archive = aclass.open(abspath) checked = None container = None pathwin = PathWin(self) exitstatus, s = pathwin.show( "Extract to " "(press ENTER for confirmation or ESC to cancel):" ) pathwin.close() logging.info("window exitstatus: {0}, '{1}'".format( exitstatus, s )) if exitstatus != 0: continue workwin = WorkWin(self) workwin.show("Extracting ...") aclass.extract(container, archive, s, checked=checked) workwin.close() TextWin(self).show("Extracted to:\n{0}".format(s)) elif self.ch in [ord('?'), curses.KEY_F1]: curses.curs_set(0) textwin = TextWin(self) textwin.show(HELP_STRING) if self.ch == ord('h'): if self.show_hiddens == True: self.show_hiddens = False else: self.show_hiddens = True self.chdir(self.area.abspath) if self.ch != -1: self.refresh_scr() if self.kill: break def show_items_warning(self): questionwin = QuestionWin(self) return questionwin.show( "There are more than {0} items in this folder," "\ndo you really want to select it?".format( ITEMS_WARNING ) ) def cancel(self): self.kill = True
name, abspath ) def __len__(self): return self.last - self.first + 1 def print_area(path): area = ViewArea(path, 4) off = 8 area.set_params(4, offset=off) print "{0} of {1}, offset:{2}, path: '{3}'".format(len(area), len(area.list), off, area.abspath) for item in area: print item if __name__ == "__main__": tree = DirectoryTree("/home/matej/workarea/matejc.myportal/src") a1 = tree.add("a1") a2 = tree.add("/home/matej/workarea/matejc.myportal/" "src/matejc/myportal/profiles", True) a3 = tree.add( "/home/matej/workarea/matejc.myportal/src/matejc/__init__.py" ) print_area("/home/matej/workarea/matejc.myportal/src/") print_area("/home/matej/workarea/matejc.myportal/src/matejc/myportal")
class LibArchive(Container, Archive): def __init__(self, path): self.path = s2u(os.path.abspath(path)) self.tree = DirectoryTree(self.path, self) with LibArchive.open(self.path) as larchive: self.archive = larchive a = larchive._a while True: try: e = _libarchive.archive_entry_new() r = _libarchive.archive_read_next_header2(a, e) if r != _libarchive.ARCHIVE_OK: break name = _libarchive.archive_entry_pathname(e) pathname = s2u(name) if pathname[-1] == '/': pathname = pathname[:-1] self.tree.add(os.path.join(self.path, pathname)) except UnicodeDecodeError: logging.info("Unreadable file name: {0} (in '{1}')".format( name, self.path )) finally: _libarchive.archive_entry_free(e) def listdir(self, path): children = self.tree[path].children return [c.data for c in children] def isenterable(self, path): children = self.tree[path].children return True if children else False def abspath(self, path): return self.tree[path].get_path() def count_items(self, path, stop_at=-1): node = self.tree[path] n = 0 for p in node.__iter__(): n += 1 if n == stop_at: break return n @staticmethod def isarchive(path): try: return libarchive.is_archive(path) except: return libarchive.is_archive_name(path) @staticmethod def open(path): return libarchive.Archive(path) @staticmethod @utf8_args(0, 1) def verify(archive_path, pathname, checked): # do not allow to go up in the directory tree like that if pathname.startswith('..'): return False # do not allow absolute paths if pathname[0] == '/': return False # if none is checked, that means, extract all if not checked: return True # if file is checked, extract it try: if "/".join([archive_path, pathname]) in checked: return True except OutOfRange: # this is expected return False # return false if file is not checked return False @staticmethod @utf8_args(2) def extract(container, archive, target_path, checked=None): target_path = os.path.abspath(target_path).encode('utf8') archive_path = s2u(archive.filename) # reopen archive file archive.denit() archive.f.seek(0) archive.init() a = archive._a try: if checked: logging.info(u"START extract selective '{0}'".format( archive_path )) while True: try: e = _libarchive.archive_entry_new() r = _libarchive.archive_read_next_header2(a, e) if r != _libarchive.ARCHIVE_OK: break pathname = _libarchive.archive_entry_pathname(e) \ .decode('utf8', 'replace') if pathname[-1] == '/': pathname = pathname[:-1] path = s2u(os.path.join( target_path, pathname )) logging.info(u"from '{0}' to '{1}'".format( pathname, path )) if LibArchive.verify(archive_path, pathname, checked): ftype = _libarchive.archive_entry_filetype(e) if stat.S_ISDIR(ftype): makepath(path) else: makepath(os.path.dirname(path)) with open(path, 'wb') as f: _libarchive.archive_read_data_into_fd( a, f.fileno() ) finally: _libarchive.archive_entry_free(e) logging.info(u"END extract selective '{0}'".format( archive_path )) else: logging.info(u"START extract all '{0}'".format(archive_path)) while True: try: e = _libarchive.archive_entry_new() r = _libarchive.archive_read_next_header2(a, e) if r != _libarchive.ARCHIVE_OK: break pathname = _libarchive.archive_entry_pathname(e) \ .decode('utf8', 'replace') path = s2u(os.path.join(target_path, pathname)) logging.info(u"from '{0}' to '{1}'".format( pathname, path )) if stat.S_ISDIR(_libarchive.archive_entry_filetype(e)): makepath(path) else: makepath(os.path.dirname(path)) with open(path, 'wb') as f: _libarchive.archive_read_data_into_fd( a, f.fileno() ) finally: _libarchive.archive_entry_free(e) logging.info(u"END extract all '{0}'".format(archive_path)) finally: archive.close() @staticmethod @utf8_args(1) def create2(container, archivepath, checked): if not isinstance(container, FileSystem): logging.info("container '{0}' is not FileSystem".format(container)) return False archivepath = os.path.abspath(archivepath) with libarchive.Archive(archivepath, 'w') as a: logging.info(u"START create selective '{0}'".format( archivepath )) for node in checked: path = s2u(node.get_path()) pathname = s2u(os.path.join(*node.get_data_array()[1:])) logging.info(u"from '{0}' to '{1}'".format( path, pathname )) a.writepath(path, pathname) logging.info(u"END create selective '{0}'".format(archivepath)) return True @staticmethod @utf8_args(1) def create(container, archivepath, checked): if not isinstance(container, FileSystem): logging.info("container '{0}' is not FileSystem".format(container)) return False archivepath = os.path.abspath(archivepath) BUFFER_SIZE = 2048 with libarchive.Archive(archivepath, 'w') as larchive: a = larchive._a for node in checked: if node.is_dir(): continue path = s2u(node.get_path()) pathname = os.path.join(*node.get_data_array()[1:]) if isinstance(pathname, unicode): pathname = pathname.encode('utf8', errors='replace') st = os.stat(path) entry = _libarchive.archive_entry_new() _libarchive.archive_entry_set_pathname(entry, pathname) _libarchive.archive_entry_set_size(entry, st.st_size) _libarchive.archive_entry_set_filetype( entry, stat.S_IFMT(st.st_mode) ) _libarchive.archive_entry_set_mtime(entry, st.st_mtime, 0) _libarchive.archive_entry_set_perm(entry, stat.S_IMODE(st.st_mode)) _libarchive.archive_write_header(a, entry) f = open(path, 'r') data = f.read(BUFFER_SIZE) count = len(data) while count > 0: _libarchive.archive_write_data_from_str(a, data) data = f.read(BUFFER_SIZE) count = len(data) f.close() _libarchive.archive_entry_free(entry) _libarchive.archive_write_close(a)
def test_add_dir(self): tree = DirectoryTree(self.testdatapath, self.fs) dir1 = self.fs.join(self.testdatapath, 'a', 'ab') tree.add(dir1) self.assertIn(dir1, tree)
class Main(object): def __init__(self, mainscr, stdscr, directory): logging.basicConfig(filename='tarman.log', filemode='w', level=logging.DEBUG) self.header_lns = 1 self.mainscr = mainscr self.stdscr = stdscr self.color = curses.has_colors() if self.color: curses.init_pair(1, curses.COLOR_BLUE, -1) self.attr_folder = curses.color_pair(1) | curses.A_BOLD curses.init_pair(2, 7, -1) self.attr_norm = curses.color_pair(2) self.kill = False self.ch = -1 self.visited = {} # TODO self.area = None self.container = FileSystem() self.directory = self.container.abspath(directory) self.checked = DirectoryTree(self.directory, self.container) self.chdir(self.directory) def header(self, text, line=0): self.mainscr.clear() self.mainscr.addstr(line, 0, text) self.mainscr.refresh() def identify_container(self, path): if self.container.isenterable(path): # is folder return self.container # force one-level archive browsing if not isinstance(self.container, FileSystem): return None return container(path) def chdir(self, newpath): if newpath is None: return False if not newpath.startswith(self.directory): return False try: if self.area is None: oldsel = 0 oldpath = self.directory else: oldsel = self.area.selected oldpath = self.area.abspath oldcontainer = self.container oldchecked = self.checked if newpath in self.visited: newsel, newcontainer, newchecked = self.visited[newpath] else: newcontainer = self.identify_container(newpath) if newcontainer is None: return False newchecked = DirectoryTree(newpath, newcontainer) newsel = 0 self.visited[oldpath] = [oldsel, oldcontainer, oldchecked] logging.info("OLD - {0} - {1} - {2}".format(oldpath, oldsel, oldcontainer.__class__.__name__)) logging.info("NEW - {0} - {1} - {2}".format(newpath, newsel, newcontainer.__class__.__name__)) h, w = self.stdscr.getmaxyx() self.container = newcontainer self.checked = newchecked self.area = ViewArea(newpath, h, newcontainer) self.header("({0}) {1}".format( self.container.__class__.__name__, self.area.abspath )) self.area.set_params(h, offset=newsel) self.refresh_scr() return True except OutOfRange: curses.flash() def insert_line(self, y, item): i, name, abspath = item self.stdscr.addstr( y, 0, "[{0}]".format('*' if abspath in self.checked else ' ') ) if self.color: if self.container.isenterable(abspath): attr = self.attr_folder name = "{0}/".format(name) else: attr = self.attr_norm self.stdscr.addstr(y, 5, name, attr) else: if self.container.isenterable(abspath): name = "{0}/".format(name) self.stdscr.addstr(y, 5, name) def refresh_scr(self): self.stdscr.clear() if len(self.area) == 0: self.stdscr.addstr(1, 5, "Directory is empty!") return iitem = 0 for item in self.area: self.insert_line(iitem, item) iitem += 1 y = self.area.selected_local h, w = self.stdscr.getmaxyx() self.stdscr.chgat(y, 0, w, curses.A_REVERSE) self.stdscr.move(y, 1) def loop(self): while not self.kill: self.ch = self.stdscr.getch() h, w = self.stdscr.getmaxyx() if self.ch in [ord('q'), 27]: self.kill = True elif self.ch == curses.KEY_UP: self.area.set_params(h, offset=-1) elif self.ch == curses.KEY_DOWN: self.area.set_params(h, offset=1) elif self.ch == curses.KEY_PPAGE: self.area.set_params(h, offset=-5) elif self.ch == curses.KEY_NPAGE: self.area.set_params(h, offset=5) elif self.ch == 32: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if abspath in self.checked: del self.checked[abspath] else: self.checked.add(abspath, sub=True) elif self.ch == curses.KEY_RIGHT: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if not self.chdir(abspath): curses.flash() elif self.ch == curses.KEY_LEFT: if not self.chdir( self.container.dirname(self.area.abspath) ): curses.flash() elif self.ch in [ord('e'), ord('E')]: if isinstance(self.container, Archive): aclass = self.container.__class__ archive = self.container.archive checked = self.checked else: index = self.area.selected if index == -1: curses.flash() continue abspath = self.area.get_abspath(index) if not abspath: curses.flash() continue aclass = get_archive_class(abspath) archive = aclass.open(abspath) checked = None aclass.extract(archive, '.', checked=checked) if self.ch != -1: self.refresh_scr() if self.kill: break self.stdscr.clear() def cancel(self): self.kill = True
def __len__(self): return self.last - self.first + 1 def print_area(path): area = ViewArea(path, 4) off = 8 area.set_params(4, offset=off) print "{0} of {1}, offset:{2}, path: '{3}'".format( len(area), len(area.list), off, area.abspath ) for item in area: print item if __name__ == "__main__": tree = DirectoryTree("/home/matej/workarea/matejc.myportal/src") a1 = tree.add("a1") a2 = tree.add( "/home/matej/workarea/matejc.myportal/" "src/matejc/myportal/profiles", True ) a3 = tree.add( "/home/matej/workarea/matejc.myportal/src/matejc/__init__.py" ) print_area("/home/matej/workarea/matejc.myportal/src/") print_area("/home/matej/workarea/matejc.myportal/src/matejc/myportal")