Пример #1
0
    def get_resources(self,
                      fs_urls,
                      dirs_only=False,
                      files_only=False,
                      single=False):

        fs_paths = [self.open_fs(fs_url) for fs_url in fs_urls]
        resources = []

        for fs, path in fs_paths:
            if path and iswildcard(path):
                if not files_only:
                    dir_paths = fs.listdir(wildcard=path, dirs_only=True)
                    for path in dir_paths:
                        resources.append([fs, path, True])
                if not dirs_only:
                    file_paths = fs.listdir(wildcard=path, files_only=True)
                    for path in file_paths:
                        resources.append([fs, path, False])
            else:
                path = path or '/'
                is_dir = fs.isdir(path)
                resource = [fs, path, is_dir]
                if not files_only and not dirs_only:
                    resources.append(resource)
                elif files_only and not is_dir:
                    resources.append(resource)
                elif dirs_only and is_dir:
                    resources.append(resource)

            if single:
                break

        return resources
Пример #2
0
 def get_resources(self, fs_urls, dirs_only=False, files_only=False, single=False):
     
     fs_paths = [self.open_fs(fs_url) for fs_url in fs_urls]                
     resources = []
     
     for fs, path in fs_paths:            
         if path and iswildcard(path):
             if not files_only:
                 dir_paths = fs.listdir(wildcard=path, dirs_only=True)
                 for path in dir_paths:
                     resources.append([fs, path, True])                        
             if not dirs_only:                    
                 file_paths = fs.listdir(wildcard=path, files_only=True)
                 for path in file_paths:
                     resources.append([fs, path, False])                        
         else:
             path = path or '/'
             is_dir = fs.isdir(path)
             resource = [fs, path, is_dir]
             if not files_only and not dirs_only:
                 resources.append(resource)
             elif files_only and not is_dir:
                 resources.append(resource)
             elif dirs_only and is_dir:
                 resources.append(resource)                
                 
         if single:
             break
         
     return resources                
Пример #3
0
 def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False):
     """Parses a FS url and returns an fs object a path within that FS object
     (if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
     
     :param fs_url: an FS url
     :param default_fs_name: the default FS to use if none is indicated (defaults is OSFS)
     :param writeable: if True, a writeable FS will be returned
     :oaram create_dir: if True, then the directory in the FS will be created
     
     """
                
     orig_url = fs_url     
     match = self.split_segments(fs_url)
     
     if match:
         fs_name, fs_url, _, path = match.groups()
         path = path or ''
         fs_url = fs_url or ''
         if ':' in fs_name:
             fs_name, sub_protocol = fs_name.split(':', 1)
             fs_url = '%s://%s' % (sub_protocol, fs_url)
         if '!' in path:
             paths = path.split('!')
             path = paths.pop()
             fs_url = '%s!%s' % (fs_url, '!'.join(paths))
         
         fs_name = fs_name or self.default_opener                                                                    
             
     else:
         fs_name = default_fs_name or self.default_opener
         fs_url = _expand_syspath(fs_url) 
         path = ''           
 
     fs_name,  fs_name_params = _parse_name(fs_name)        
     opener = self.get_opener(fs_name)
     
     if fs_url is None:
         raise OpenerError("Unable to parse '%s'" % orig_url)        
     
     fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
     
     if fs_path and iswildcard(fs_path):
         pathname, resourcename = pathsplit(fs_path or '')
         if pathname:
             fs = fs.opendir(pathname)
         return fs, resourcename
                         
     fs_path = join(fs_path, path)
     
     if create_dir and fs_path:
         fs.makedir(fs_path, allow_recreate=True)     
             
     pathname, resourcename = pathsplit(fs_path or '')        
     if pathname and resourcename:
         fs = fs.opendir(pathname)
         fs_path = resourcename
                            
     return fs, fs_path or ''        
Пример #4
0
 def test_iswildcard(self):
     self.assertTrue(iswildcard("*"))
     self.assertTrue(iswildcard("*.jpg"))
     self.assertTrue(iswildcard("foo/*"))
     self.assertTrue(iswildcard("foo/{}"))
     self.assertFalse(iswildcard("foo"))
     self.assertFalse(iswildcard("img.jpg"))
     self.assertFalse(iswildcard("foo/bar"))
Пример #5
0
    def expand_wildcard(self, fs, path):
        if path is None:
            return [], []
        pathname, resourcename = pathsplit(path)
        if iswildcard(resourcename):
            dir_paths = fs.listdir(pathname, wildcard=resourcename, absolute=True, dirs_only=True)

            file_paths = fs.listdir(pathname, wildcard=resourcename, absolute=True, files_only=True)
            return dir_paths, file_paths

        else:
            if fs.isdir(path):
                # file_paths = fs.listdir(path,
                #                        absolute=True)
                return [path], []
            return [], [path]
Пример #6
0
    def expand_wildcard(self, fs, path):
        if path is None:
            return [], []
        pathname, resourcename = pathsplit(path)
        if iswildcard(resourcename):
            dir_paths = fs.listdir(pathname,
                                   wildcard=resourcename,
                                   absolute=True,
                                   dirs_only=True)

            file_paths = fs.listdir(pathname,
                                    wildcard=resourcename,
                                    absolute=True,
                                    files_only=True)
            return dir_paths, file_paths

        else:
            if fs.isdir(path):
                #file_paths = fs.listdir(path,
                #                        absolute=True)
                return [path], []
            return [], [path]
Пример #7
0
    def do_run(self, options, args):
        output = self.output

        if not args:
            args = [u'.']

        dir_paths = []
        file_paths = []
        fs_used = set()
        for fs_url in args:
            fs, path = self.open_fs(fs_url)
            fs_used.add(fs)
            path = path or '.'
            wildcard = None

            if iswildcard(path):
                path, wildcard = pathsplit(path)

            if path != '.' and fs.isfile(path):
                if not options.dirsonly:
                    file_paths.append(path)
            else:
                if not options.filesonly:
                    dir_paths += fs.listdir(path,
                                            wildcard=wildcard,
                                            full=options.fullpath
                                            or options.url,
                                            dirs_only=True)

                if not options.dirsonly:
                    file_paths += fs.listdir(path,
                                             wildcard=wildcard,
                                             full=options.fullpath
                                             or options.url,
                                             files_only=True)

        for fs in fs_used:
            try:
                fs.close()
            except FSError:
                pass

        if options.syspath:
            # Path without a syspath, just won't be displayed
            dir_paths = filter(
                None,
                [fs.getsyspath(path, allow_none=True) for path in dir_paths])
            file_paths = filter(
                None,
                [fs.getsyspath(path, allow_none=True) for path in file_paths])

        if options.url:
            # Path without a syspath, just won't be displayed
            dir_paths = filter(
                None,
                [fs.getpathurl(path, allow_none=True) for path in dir_paths])
            file_paths = filter(
                None,
                [fs.getpathurl(path, allow_none=True) for path in file_paths])

        dirs = frozenset(dir_paths)
        paths = sorted(dir_paths + file_paths, key=lambda p: p.lower())

        if not options.all:
            paths = [path for path in paths if not isdotfile(path)]

        if not paths:
            return

        def columnize(paths, num_columns):

            col_height = (len(paths) + num_columns - 1) / num_columns
            columns = [[] for _ in xrange(num_columns)]
            col_no = 0
            col_pos = 0
            for path in paths:
                columns[col_no].append(path)
                col_pos += 1
                if col_pos >= col_height:
                    col_no += 1
                    col_pos = 0

            padded_columns = []

            wrap_filename = self.wrap_filename
            wrap_dirname = self.wrap_dirname

            def wrap(path):
                if path in dirs:
                    return wrap_dirname(path.ljust(max_width))
                else:
                    return wrap_filename(path.ljust(max_width))

            for column in columns:
                if column:
                    max_width = max([len(path) for path in column])
                else:
                    max_width = 1
                max_width = min(max_width, terminal_width)
                padded_columns.append([wrap(path) for path in column])

            return padded_columns

        def condense_columns(columns):
            max_column_height = max([len(col) for col in columns])
            lines = [[] for _ in xrange(max_column_height)]
            for column in columns:
                for line, path in zip(lines, column):
                    line.append(path)
            return '\n'.join(u'  '.join(line) for line in lines)

        if options.long:
            for path in paths:
                if path in dirs:
                    output((self.wrap_dirname(path), '\n'))
                else:
                    output((self.wrap_filename(path), '\n'))

        else:
            terminal_width = self.terminal_width
            path_widths = [len(path) for path in paths]
            smallest_paths = min(path_widths)
            num_paths = len(paths)

            num_cols = min(terminal_width / (smallest_paths + 2), num_paths)
            while num_cols:
                col_height = (num_paths + num_cols - 1) / num_cols
                line_width = 0
                for col_no in xrange(num_cols):
                    try:
                        col_width = max(
                            path_widths[col_no * col_height:(col_no + 1) *
                                        col_height])
                    except ValueError:
                        continue
                    line_width += col_width
                    if line_width > terminal_width:
                        break
                    line_width += 2
                else:
                    if line_width - 1 <= terminal_width:
                        break
                num_cols -= 1
            num_cols = max(1, num_cols)
            columns = columnize(paths, num_cols)
            output((condense_columns(columns), '\n'))
Пример #8
0
    def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False, cache_hint=True):
        """Parses a FS url and returns an fs object a path within that FS object
        (if indicated in the path). A tuple of (<FS instance>, <path>) is returned.

        :param fs_url: an FS url
        :param default_fs_name: the default FS to use if none is indicated (defaults is OSFS)
        :param writeable: if True, a writeable FS will be returned
        :param create_dir: if True, then the directory in the FS will be created

        """

        orig_url = fs_url
        match = self.split_segments(fs_url)

        if match:
            fs_name, credentials, url1, url2, path = match.groups()
            if credentials:
                fs_url = '%s@%s' % (credentials, url1)
            else:
                fs_url = url2
            path = path or ''
            fs_url = fs_url or ''
            if ':' in fs_name:
                fs_name, sub_protocol = fs_name.split(':', 1)
                fs_url = '%s://%s' % (sub_protocol, fs_url)
            if '!' in path:
                paths = path.split('!')
                path = paths.pop()
                fs_url = '%s!%s' % (fs_url, '!'.join(paths))

            fs_name = fs_name or self.default_opener
        else:
            fs_name = default_fs_name or self.default_opener
            fs_url = _expand_syspath(fs_url)
            path = ''

        fs_name,  fs_name_params = _parse_name(fs_name)
        opener = self.get_opener(fs_name)

        if fs_url is None:
            raise OpenerError("Unable to parse '%s'" % orig_url)

        fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
        fs.cache_hint(cache_hint)

        if fs_path and iswildcard(fs_path):
            pathname, resourcename = pathsplit(fs_path or '')
            if pathname:
                fs = fs.opendir(pathname)
            return fs, resourcename

        fs_path = join(fs_path, path)

        if create_dir and fs_path:
            if not fs.getmeta('read_only', False):
                fs.makedir(fs_path, allow_recreate=True)

        pathname, resourcename = pathsplit(fs_path or '')
        if pathname and resourcename:
            fs = fs.opendir(pathname)
            fs_path = resourcename

        return fs, fs_path or ''
Пример #9
0
    def do_run(self, options, args):

        self.options = options
        if len(args) < 2:
            self.error("at least two filesystems required\n")
            return 1

        srcs = args[:-1]
        dst = args[-1]

        dst_fs, dst_path = self.open_fs(dst, writeable=True, create_dir=True)

        if dst_path is not None and dst_fs.isfile(dst_path):
            self.error('Destination must be a directory\n')
            return 1

        if dst_path:
            dst_fs = dst_fs.makeopendir(dst_path)
            dst_path = None

        copy_fs_paths = []

        progress = options.progress

        if progress:
            sys.stdout.write(self.progress_bar(len(srcs), 0, 'scanning...'))
            sys.stdout.flush()

        self.root_dirs = []
        for i, fs_url in enumerate(srcs):
            src_fs, src_path = self.open_fs(fs_url)

            if src_path is None:
                src_path = '/'

            if iswildcard(src_path):
                for file_path in src_fs.listdir(wildcard=src_path, full=True):
                    copy_fs_paths.append(
                        (self.FILE, src_fs, file_path, file_path))

            else:
                if src_fs.isdir(src_path):
                    self.root_dirs.append((src_fs, src_path))
                    src_sub_fs = src_fs.opendir(src_path)
                    for dir_path, file_paths in src_sub_fs.walk():
                        if dir_path not in ('', '/'):
                            copy_fs_paths.append(
                                (self.DIR, src_sub_fs, dir_path, dir_path))
                        sub_fs = src_sub_fs.opendir(dir_path)
                        for file_path in file_paths:
                            copy_fs_paths.append(
                                (self.FILE, sub_fs, file_path,
                                 pathjoin(dir_path, file_path)))
                else:
                    if src_fs.exists(src_path):
                        copy_fs_paths.append(
                            (self.FILE, src_fs, src_path, src_path))
                    else:
                        self.error('%s is not a file or directory\n' %
                                   src_path)
                        return 1

            if progress:
                sys.stdout.write(
                    self.progress_bar(len(srcs), i + 1, 'scanning...'))
                sys.stdout.flush()

        if progress:
            sys.stdout.write(
                self.progress_bar(len(copy_fs_paths), 0, self.get_verb()))
            sys.stdout.flush()

        if self.options.threads > 1:
            copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR]
            copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE]
            for path_type, fs, path, dest_path in copy_fs_dirs:
                dst_fs.makedir(path, allow_recreate=True, recursive=True)

        self.lock = threading.RLock()

        self.total_files = len(copy_fs_paths)
        self.done_files = 0

        file_queue = queue.Queue()
        threads = [
            FileOpThread(self.get_action(), 'T%i' % i, dst_fs, file_queue,
                         self.on_done, self.on_error)
            for i in xrange(options.threads)
        ]

        for thread in threads:
            thread.start()

        self.action_errors = []
        complete = False
        try:
            enqueue = file_queue.put
            for resource in copy_fs_paths:
                enqueue(resource)

            while not file_queue.empty():
                time.sleep(0)
                if self.any_error():
                    raise SystemExit
            # Can't use queue.join here, or KeyboardInterrupt will not be
            # caught until the queue is finished
            #file_queue.join()

        except KeyboardInterrupt:
            options.progress = False
            self.output("\nCancelling...\n")

        except SystemExit:
            options.progress = False

        finally:
            sys.stdout.flush()
            for thread in threads:
                thread.finish_event.set()
            for thread in threads:
                thread.join()
            complete = True
            if not self.any_error():
                self.post_actions()

        dst_fs.close()

        if self.action_errors:
            for error in self.action_errors:
                self.error(self.wrap_error(unicode(error)) + '\n')
            sys.stdout.flush()
        else:
            if complete and options.progress:
                sys.stdout.write(
                    self.progress_bar(self.total_files, self.done_files, ''))
                sys.stdout.write('\n')
                sys.stdout.flush()
Пример #10
0
    def do_run(self, options, args):
        output = self.output

        if not args:
            args = [u'.']

        dir_paths = []
        file_paths = []
        fs_used = set()
        for fs_url in args:
            fs, path = self.open_fs(fs_url)
            fs_used.add(fs)
            path = path or '.'
            wildcard = None

            if iswildcard(path):
                path, wildcard = pathsplit(path)

            if path != '.' and fs.isfile(path):
                if not options.dirsonly:
                    file_paths.append(path)
            else:
                if not options.filesonly:
                    dir_paths += fs.listdir(path,
                                            wildcard=wildcard,
                                            full=options.fullpath or options.url,
                                            dirs_only=True)

                if not options.dirsonly:
                    file_paths += fs.listdir(path,
                                             wildcard=wildcard,
                                             full=options.fullpath or options.url,
                                             files_only=True)

        for fs in fs_used:
            try:
                fs.close()
            except FSError:
                pass

        if options.syspath:
            # Path without a syspath, just won't be displayed
            dir_paths = filter(None, [fs.getsyspath(path, allow_none=True) for path in dir_paths])
            file_paths = filter(None, [fs.getsyspath(path, allow_none=True) for path in file_paths])

        if options.url:
            # Path without a syspath, just won't be displayed
            dir_paths = filter(None, [fs.getpathurl(path, allow_none=True) for path in dir_paths])
            file_paths = filter(None, [fs.getpathurl(path, allow_none=True) for path in file_paths])

        dirs = frozenset(dir_paths)
        paths = sorted(dir_paths + file_paths, key=lambda p: p.lower())

        if not options.all:
            paths = [path for path in paths if not isdotfile(path)]

        if not paths:
            return

        def columnize(paths, num_columns):

            col_height = (len(paths) + num_columns - 1) / num_columns
            columns = [[] for _ in xrange(num_columns)]
            col_no = 0
            col_pos = 0
            for path in paths:
                columns[col_no].append(path)
                col_pos += 1
                if col_pos >= col_height:
                    col_no += 1
                    col_pos = 0

            padded_columns = []

            wrap_filename = self.wrap_filename
            wrap_dirname = self.wrap_dirname

            def wrap(path):
                if path in dirs:
                    return wrap_dirname(path.ljust(max_width))
                else:
                    return wrap_filename(path.ljust(max_width))

            for column in columns:
                if column:
                    max_width = max([len(path) for path in column])
                else:
                    max_width = 1
                max_width = min(max_width, terminal_width)
                padded_columns.append([wrap(path) for path in column])

            return padded_columns

        def condense_columns(columns):
            max_column_height = max([len(col) for col in columns])
            lines = [[] for _ in xrange(max_column_height)]
            for column in columns:
                for line, path in zip(lines, column):
                    line.append(path)
            return '\n'.join(u'  '.join(line) for line in lines)

        if options.long:
            for path in paths:
                if path in dirs:
                    output((self.wrap_dirname(path), '\n'))
                else:
                    output((self.wrap_filename(path), '\n'))

        else:
            terminal_width = self.terminal_width
            path_widths = [len(path) for path in paths]
            smallest_paths = min(path_widths)
            num_paths = len(paths)

            num_cols = min(terminal_width // (smallest_paths + 2), num_paths)
            while num_cols:
                col_height = (num_paths + num_cols - 1) // num_cols
                line_width = 0
                for col_no in xrange(num_cols):
                    try:
                        col_width = max(path_widths[col_no * col_height: (col_no + 1) * col_height])
                    except ValueError:
                        continue
                    line_width += col_width
                    if line_width > terminal_width:
                        break
                    line_width += 2
                else:
                    if line_width - 1 <= terminal_width:
                        break
                num_cols -= 1
            num_cols = max(1, num_cols)
            columns = columnize(paths, num_cols)
            output((condense_columns(columns), '\n'))
Пример #11
0
    def do_run(self, options, args):
               
        self.options = options 
        if len(args) < 2:
            self.error("at least two filesystems required\n")
            return 1
        
        srcs = args[:-1]
        dst = args[-1] 
        
        dst_fs, dst_path = self.open_fs(dst, writeable=True, create_dir=True)
        
        if dst_path is not None and dst_fs.isfile(dst_path):
            self.error('Destination must be a directory\n')
            return 1
        
        if dst_path:
            dst_fs = dst_fs.makeopendir(dst_path)
            dst_path = None                      
                
        copy_fs_paths = []
        
        progress = options.progress                
        
        if progress:
            sys.stdout.write(self.progress_bar(len(srcs), 0, 'scanning...'))
            sys.stdout.flush()
        
        self.root_dirs = [] 
        for i, fs_url in enumerate(srcs):
            src_fs, src_path = self.open_fs(fs_url)                      

            if src_path is None:
                src_path = '/'

            if iswildcard(src_path):
                for file_path in src_fs.listdir(wildcard=src_path, full=True):
                    copy_fs_paths.append((self.FILE, src_fs, file_path, file_path))
                    
            else:                
                if src_fs.isdir(src_path): 
                    self.root_dirs.append((src_fs, src_path))                                        
                    src_sub_fs = src_fs.opendir(src_path)
                    for dir_path, file_paths in src_sub_fs.walk():
                        if dir_path not in ('', '/'):                                            
                            copy_fs_paths.append((self.DIR, src_sub_fs, dir_path, dir_path))
                        sub_fs = src_sub_fs.opendir(dir_path)
                        for file_path in file_paths:                                                         
                            copy_fs_paths.append((self.FILE, sub_fs, file_path, pathjoin(dir_path, file_path)))
                else:
                    if src_fs.exists(src_path):
                        copy_fs_paths.append((self.FILE, src_fs, src_path, src_path))
                    else:
                        self.error('%s is not a file or directory\n' % src_path)
                        return 1 
                    
            if progress:
                sys.stdout.write(self.progress_bar(len(srcs), i + 1, 'scanning...'))
                sys.stdout.flush()
                                       
        if progress:
            sys.stdout.write(self.progress_bar(len(copy_fs_paths), 0, self.get_verb()))
            sys.stdout.flush()
                                        
        if self.options.threads > 1:            
            copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR]
            copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE]            
            for path_type, fs, path, dest_path in copy_fs_dirs:               
                dst_fs.makedir(path, allow_recreate=True, recursive=True)                             
        
        self.lock = threading.RLock()
            
        self.total_files = len(copy_fs_paths)
        self.done_files = 0
                
        file_queue = queue.Queue()        
        threads = [FileOpThread(self.get_action(),
                                'T%i' % i,
                                dst_fs,
                                file_queue,
                                self.on_done,
                                self.on_error)
                        for i in xrange(options.threads)]
        
        for thread in threads:
            thread.start()
        
        self.action_errors = []
        complete = False
        try:        
            enqueue = file_queue.put            
            for resource in copy_fs_paths:
                enqueue(resource)
                        
            while not file_queue.empty():
                time.sleep(0)
                if self.any_error():
                    raise SystemExit
            # Can't use queue.join here, or KeyboardInterrupt will not be
            # caught until the queue is finished 
            #file_queue.join()
        
        except KeyboardInterrupt:            
            options.progress = False                    
            self.output("\nCancelling...\n")
                
        except SystemExit:
            options.progress = False            
                               
        finally:
            sys.stdout.flush()                
            for thread in threads:
                thread.finish_event.set()                           
            for thread in threads:
                thread.join()
            complete = True
            if not self.any_error():
                self.post_actions()
                                   
        dst_fs.close()
        
        if self.action_errors:
            for error in self.action_errors:
                self.error(self.wrap_error(unicode(error)) + '\n')            
            sys.stdout.flush()
        else:
            if complete and options.progress:
                sys.stdout.write(self.progress_bar(self.total_files, self.done_files, ''))
                sys.stdout.write('\n')
                sys.stdout.flush()