def get_resources(self,
fs_urls,
dirs_only=False,
files_only=False,
single=False):
fs_paths = [self.open_fs(fs_url) for fs_url in fs_urls]
resources = []
for fs, path in fs_paths:
if path and iswildcard(path):
if not files_only:
dir_paths = fs.listdir(wildcard=path, dirs_only=True)
for path in dir_paths:
resources.append([fs, path, True])
if not dirs_only:
file_paths = fs.listdir(wildcard=path, files_only=True)
for path in file_paths:
resources.append([fs, path, False])
else:
path = path or '/'
is_dir = fs.isdir(path)
resource = [fs, path, is_dir]
if not files_only and not dirs_only:
resources.append(resource)
elif files_only and not is_dir:
resources.append(resource)
elif dirs_only and is_dir:
resources.append(resource)
if single:
break
return resources
def get_resources(self, fs_urls, dirs_only=False, files_only=False, single=False):
fs_paths = [self.open_fs(fs_url) for fs_url in fs_urls]
resources = []
for fs, path in fs_paths:
if path and iswildcard(path):
if not files_only:
dir_paths = fs.listdir(wildcard=path, dirs_only=True)
for path in dir_paths:
resources.append([fs, path, True])
if not dirs_only:
file_paths = fs.listdir(wildcard=path, files_only=True)
for path in file_paths:
resources.append([fs, path, False])
else:
path = path or '/'
is_dir = fs.isdir(path)
resource = [fs, path, is_dir]
if not files_only and not dirs_only:
resources.append(resource)
elif files_only and not is_dir:
resources.append(resource)
elif dirs_only and is_dir:
resources.append(resource)
if single:
break
return resources
def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False):
"""Parses a FS url and returns an fs object a path within that FS object
(if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
:param fs_url: an FS url
:param default_fs_name: the default FS to use if none is indicated (defaults is OSFS)
:param writeable: if True, a writeable FS will be returned
:oaram create_dir: if True, then the directory in the FS will be created
"""
orig_url = fs_url
match = self.split_segments(fs_url)
if match:
fs_name, fs_url, _, path = match.groups()
path = path or ''
fs_url = fs_url or ''
if ':' in fs_name:
fs_name, sub_protocol = fs_name.split(':', 1)
fs_url = '%s://%s' % (sub_protocol, fs_url)
if '!' in path:
paths = path.split('!')
path = paths.pop()
fs_url = '%s!%s' % (fs_url, '!'.join(paths))
fs_name = fs_name or self.default_opener
else:
fs_name = default_fs_name or self.default_opener
fs_url = _expand_syspath(fs_url)
path = ''
fs_name, fs_name_params = _parse_name(fs_name)
opener = self.get_opener(fs_name)
if fs_url is None:
raise OpenerError("Unable to parse '%s'" % orig_url)
fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
if fs_path and iswildcard(fs_path):
pathname, resourcename = pathsplit(fs_path or '')
if pathname:
fs = fs.opendir(pathname)
return fs, resourcename
fs_path = join(fs_path, path)
if create_dir and fs_path:
fs.makedir(fs_path, allow_recreate=True)
pathname, resourcename = pathsplit(fs_path or '')
if pathname and resourcename:
fs = fs.opendir(pathname)
fs_path = resourcename
return fs, fs_path or ''
def test_iswildcard(self):
self.assertTrue(iswildcard("*"))
self.assertTrue(iswildcard("*.jpg"))
self.assertTrue(iswildcard("foo/*"))
self.assertTrue(iswildcard("foo/{}"))
self.assertFalse(iswildcard("foo"))
self.assertFalse(iswildcard("img.jpg"))
self.assertFalse(iswildcard("foo/bar"))
def expand_wildcard(self, fs, path):
if path is None:
return [], []
pathname, resourcename = pathsplit(path)
if iswildcard(resourcename):
dir_paths = fs.listdir(pathname, wildcard=resourcename, absolute=True, dirs_only=True)
file_paths = fs.listdir(pathname, wildcard=resourcename, absolute=True, files_only=True)
return dir_paths, file_paths
else:
if fs.isdir(path):
# file_paths = fs.listdir(path,
# absolute=True)
return [path], []
return [], [path]
def expand_wildcard(self, fs, path):
if path is None:
return [], []
pathname, resourcename = pathsplit(path)
if iswildcard(resourcename):
dir_paths = fs.listdir(pathname,
wildcard=resourcename,
absolute=True,
dirs_only=True)
file_paths = fs.listdir(pathname,
wildcard=resourcename,
absolute=True,
files_only=True)
return dir_paths, file_paths
else:
if fs.isdir(path):
#file_paths = fs.listdir(path,
# absolute=True)
return [path], []
return [], [path]
def do_run(self, options, args):
output = self.output
if not args:
args = [u'.']
dir_paths = []
file_paths = []
fs_used = set()
for fs_url in args:
fs, path = self.open_fs(fs_url)
fs_used.add(fs)
path = path or '.'
wildcard = None
if iswildcard(path):
path, wildcard = pathsplit(path)
if path != '.' and fs.isfile(path):
if not options.dirsonly:
file_paths.append(path)
else:
if not options.filesonly:
dir_paths += fs.listdir(path,
wildcard=wildcard,
full=options.fullpath
or options.url,
dirs_only=True)
if not options.dirsonly:
file_paths += fs.listdir(path,
wildcard=wildcard,
full=options.fullpath
or options.url,
files_only=True)
for fs in fs_used:
try:
fs.close()
except FSError:
pass
if options.syspath:
# Path without a syspath, just won't be displayed
dir_paths = filter(
None,
[fs.getsyspath(path, allow_none=True) for path in dir_paths])
file_paths = filter(
None,
[fs.getsyspath(path, allow_none=True) for path in file_paths])
if options.url:
# Path without a syspath, just won't be displayed
dir_paths = filter(
None,
[fs.getpathurl(path, allow_none=True) for path in dir_paths])
file_paths = filter(
None,
[fs.getpathurl(path, allow_none=True) for path in file_paths])
dirs = frozenset(dir_paths)
paths = sorted(dir_paths + file_paths, key=lambda p: p.lower())
if not options.all:
paths = [path for path in paths if not isdotfile(path)]
if not paths:
return
def columnize(paths, num_columns):
col_height = (len(paths) + num_columns - 1) / num_columns
columns = [[] for _ in xrange(num_columns)]
col_no = 0
col_pos = 0
for path in paths:
columns[col_no].append(path)
col_pos += 1
if col_pos >= col_height:
col_no += 1
col_pos = 0
padded_columns = []
wrap_filename = self.wrap_filename
wrap_dirname = self.wrap_dirname
def wrap(path):
if path in dirs:
return wrap_dirname(path.ljust(max_width))
else:
return wrap_filename(path.ljust(max_width))
for column in columns:
if column:
max_width = max([len(path) for path in column])
else:
max_width = 1
max_width = min(max_width, terminal_width)
padded_columns.append([wrap(path) for path in column])
return padded_columns
def condense_columns(columns):
max_column_height = max([len(col) for col in columns])
lines = [[] for _ in xrange(max_column_height)]
for column in columns:
for line, path in zip(lines, column):
line.append(path)
return '\n'.join(u' '.join(line) for line in lines)
if options.long:
for path in paths:
if path in dirs:
output((self.wrap_dirname(path), '\n'))
else:
output((self.wrap_filename(path), '\n'))
else:
terminal_width = self.terminal_width
path_widths = [len(path) for path in paths]
smallest_paths = min(path_widths)
num_paths = len(paths)
num_cols = min(terminal_width / (smallest_paths + 2), num_paths)
while num_cols:
col_height = (num_paths + num_cols - 1) / num_cols
line_width = 0
for col_no in xrange(num_cols):
try:
col_width = max(
path_widths[col_no * col_height:(col_no + 1) *
col_height])
except ValueError:
continue
line_width += col_width
if line_width > terminal_width:
break
line_width += 2
else:
if line_width - 1 <= terminal_width:
break
num_cols -= 1
num_cols = max(1, num_cols)
columns = columnize(paths, num_cols)
output((condense_columns(columns), '\n'))
def parse(self, fs_url, default_fs_name=None, writeable=False, create_dir=False, cache_hint=True):
"""Parses a FS url and returns an fs object a path within that FS object
(if indicated in the path). A tuple of (<FS instance>, <path>) is returned.
:param fs_url: an FS url
:param default_fs_name: the default FS to use if none is indicated (defaults is OSFS)
:param writeable: if True, a writeable FS will be returned
:param create_dir: if True, then the directory in the FS will be created
"""
orig_url = fs_url
match = self.split_segments(fs_url)
if match:
fs_name, credentials, url1, url2, path = match.groups()
if credentials:
fs_url = '%s@%s' % (credentials, url1)
else:
fs_url = url2
path = path or ''
fs_url = fs_url or ''
if ':' in fs_name:
fs_name, sub_protocol = fs_name.split(':', 1)
fs_url = '%s://%s' % (sub_protocol, fs_url)
if '!' in path:
paths = path.split('!')
path = paths.pop()
fs_url = '%s!%s' % (fs_url, '!'.join(paths))
fs_name = fs_name or self.default_opener
else:
fs_name = default_fs_name or self.default_opener
fs_url = _expand_syspath(fs_url)
path = ''
fs_name, fs_name_params = _parse_name(fs_name)
opener = self.get_opener(fs_name)
if fs_url is None:
raise OpenerError("Unable to parse '%s'" % orig_url)
fs, fs_path = opener.get_fs(self, fs_name, fs_name_params, fs_url, writeable, create_dir)
fs.cache_hint(cache_hint)
if fs_path and iswildcard(fs_path):
pathname, resourcename = pathsplit(fs_path or '')
if pathname:
fs = fs.opendir(pathname)
return fs, resourcename
fs_path = join(fs_path, path)
if create_dir and fs_path:
if not fs.getmeta('read_only', False):
fs.makedir(fs_path, allow_recreate=True)
pathname, resourcename = pathsplit(fs_path or '')
if pathname and resourcename:
fs = fs.opendir(pathname)
fs_path = resourcename
return fs, fs_path or ''
def do_run(self, options, args):
self.options = options
if len(args) < 2:
self.error("at least two filesystems required\n")
return 1
srcs = args[:-1]
dst = args[-1]
dst_fs, dst_path = self.open_fs(dst, writeable=True, create_dir=True)
if dst_path is not None and dst_fs.isfile(dst_path):
self.error('Destination must be a directory\n')
return 1
if dst_path:
dst_fs = dst_fs.makeopendir(dst_path)
dst_path = None
copy_fs_paths = []
progress = options.progress
if progress:
sys.stdout.write(self.progress_bar(len(srcs), 0, 'scanning...'))
sys.stdout.flush()
self.root_dirs = []
for i, fs_url in enumerate(srcs):
src_fs, src_path = self.open_fs(fs_url)
if src_path is None:
src_path = '/'
if iswildcard(src_path):
for file_path in src_fs.listdir(wildcard=src_path, full=True):
copy_fs_paths.append(
(self.FILE, src_fs, file_path, file_path))
else:
if src_fs.isdir(src_path):
self.root_dirs.append((src_fs, src_path))
src_sub_fs = src_fs.opendir(src_path)
for dir_path, file_paths in src_sub_fs.walk():
if dir_path not in ('', '/'):
copy_fs_paths.append(
(self.DIR, src_sub_fs, dir_path, dir_path))
sub_fs = src_sub_fs.opendir(dir_path)
for file_path in file_paths:
copy_fs_paths.append(
(self.FILE, sub_fs, file_path,
pathjoin(dir_path, file_path)))
else:
if src_fs.exists(src_path):
copy_fs_paths.append(
(self.FILE, src_fs, src_path, src_path))
else:
self.error('%s is not a file or directory\n' %
src_path)
return 1
if progress:
sys.stdout.write(
self.progress_bar(len(srcs), i + 1, 'scanning...'))
sys.stdout.flush()
if progress:
sys.stdout.write(
self.progress_bar(len(copy_fs_paths), 0, self.get_verb()))
sys.stdout.flush()
if self.options.threads > 1:
copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR]
copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE]
for path_type, fs, path, dest_path in copy_fs_dirs:
dst_fs.makedir(path, allow_recreate=True, recursive=True)
self.lock = threading.RLock()
self.total_files = len(copy_fs_paths)
self.done_files = 0
file_queue = queue.Queue()
threads = [
FileOpThread(self.get_action(), 'T%i' % i, dst_fs, file_queue,
self.on_done, self.on_error)
for i in xrange(options.threads)
]
for thread in threads:
thread.start()
self.action_errors = []
complete = False
try:
enqueue = file_queue.put
for resource in copy_fs_paths:
enqueue(resource)
while not file_queue.empty():
time.sleep(0)
if self.any_error():
raise SystemExit
# Can't use queue.join here, or KeyboardInterrupt will not be
# caught until the queue is finished
#file_queue.join()
except KeyboardInterrupt:
options.progress = False
self.output("\nCancelling...\n")
except SystemExit:
options.progress = False
finally:
sys.stdout.flush()
for thread in threads:
thread.finish_event.set()
for thread in threads:
thread.join()
complete = True
if not self.any_error():
self.post_actions()
dst_fs.close()
if self.action_errors:
for error in self.action_errors:
self.error(self.wrap_error(unicode(error)) + '\n')
sys.stdout.flush()
else:
if complete and options.progress:
sys.stdout.write(
self.progress_bar(self.total_files, self.done_files, ''))
sys.stdout.write('\n')
sys.stdout.flush()
def do_run(self, options, args):
output = self.output
if not args:
args = [u'.']
dir_paths = []
file_paths = []
fs_used = set()
for fs_url in args:
fs, path = self.open_fs(fs_url)
fs_used.add(fs)
path = path or '.'
wildcard = None
if iswildcard(path):
path, wildcard = pathsplit(path)
if path != '.' and fs.isfile(path):
if not options.dirsonly:
file_paths.append(path)
else:
if not options.filesonly:
dir_paths += fs.listdir(path,
wildcard=wildcard,
full=options.fullpath or options.url,
dirs_only=True)
if not options.dirsonly:
file_paths += fs.listdir(path,
wildcard=wildcard,
full=options.fullpath or options.url,
files_only=True)
for fs in fs_used:
try:
fs.close()
except FSError:
pass
if options.syspath:
# Path without a syspath, just won't be displayed
dir_paths = filter(None, [fs.getsyspath(path, allow_none=True) for path in dir_paths])
file_paths = filter(None, [fs.getsyspath(path, allow_none=True) for path in file_paths])
if options.url:
# Path without a syspath, just won't be displayed
dir_paths = filter(None, [fs.getpathurl(path, allow_none=True) for path in dir_paths])
file_paths = filter(None, [fs.getpathurl(path, allow_none=True) for path in file_paths])
dirs = frozenset(dir_paths)
paths = sorted(dir_paths + file_paths, key=lambda p: p.lower())
if not options.all:
paths = [path for path in paths if not isdotfile(path)]
if not paths:
return
def columnize(paths, num_columns):
col_height = (len(paths) + num_columns - 1) / num_columns
columns = [[] for _ in xrange(num_columns)]
col_no = 0
col_pos = 0
for path in paths:
columns[col_no].append(path)
col_pos += 1
if col_pos >= col_height:
col_no += 1
col_pos = 0
padded_columns = []
wrap_filename = self.wrap_filename
wrap_dirname = self.wrap_dirname
def wrap(path):
if path in dirs:
return wrap_dirname(path.ljust(max_width))
else:
return wrap_filename(path.ljust(max_width))
for column in columns:
if column:
max_width = max([len(path) for path in column])
else:
max_width = 1
max_width = min(max_width, terminal_width)
padded_columns.append([wrap(path) for path in column])
return padded_columns
def condense_columns(columns):
max_column_height = max([len(col) for col in columns])
lines = [[] for _ in xrange(max_column_height)]
for column in columns:
for line, path in zip(lines, column):
line.append(path)
return '\n'.join(u' '.join(line) for line in lines)
if options.long:
for path in paths:
if path in dirs:
output((self.wrap_dirname(path), '\n'))
else:
output((self.wrap_filename(path), '\n'))
else:
terminal_width = self.terminal_width
path_widths = [len(path) for path in paths]
smallest_paths = min(path_widths)
num_paths = len(paths)
num_cols = min(terminal_width // (smallest_paths + 2), num_paths)
while num_cols:
col_height = (num_paths + num_cols - 1) // num_cols
line_width = 0
for col_no in xrange(num_cols):
try:
col_width = max(path_widths[col_no * col_height: (col_no + 1) * col_height])
except ValueError:
continue
line_width += col_width
if line_width > terminal_width:
break
line_width += 2
else:
if line_width - 1 <= terminal_width:
break
num_cols -= 1
num_cols = max(1, num_cols)
columns = columnize(paths, num_cols)
output((condense_columns(columns), '\n'))
def do_run(self, options, args):
self.options = options
if len(args) < 2:
self.error("at least two filesystems required\n")
return 1
srcs = args[:-1]
dst = args[-1]
dst_fs, dst_path = self.open_fs(dst, writeable=True, create_dir=True)
if dst_path is not None and dst_fs.isfile(dst_path):
self.error('Destination must be a directory\n')
return 1
if dst_path:
dst_fs = dst_fs.makeopendir(dst_path)
dst_path = None
copy_fs_paths = []
progress = options.progress
if progress:
sys.stdout.write(self.progress_bar(len(srcs), 0, 'scanning...'))
sys.stdout.flush()
self.root_dirs = []
for i, fs_url in enumerate(srcs):
src_fs, src_path = self.open_fs(fs_url)
if src_path is None:
src_path = '/'
if iswildcard(src_path):
for file_path in src_fs.listdir(wildcard=src_path, full=True):
copy_fs_paths.append((self.FILE, src_fs, file_path, file_path))
else:
if src_fs.isdir(src_path):
self.root_dirs.append((src_fs, src_path))
src_sub_fs = src_fs.opendir(src_path)
for dir_path, file_paths in src_sub_fs.walk():
if dir_path not in ('', '/'):
copy_fs_paths.append((self.DIR, src_sub_fs, dir_path, dir_path))
sub_fs = src_sub_fs.opendir(dir_path)
for file_path in file_paths:
copy_fs_paths.append((self.FILE, sub_fs, file_path, pathjoin(dir_path, file_path)))
else:
if src_fs.exists(src_path):
copy_fs_paths.append((self.FILE, src_fs, src_path, src_path))
else:
self.error('%s is not a file or directory\n' % src_path)
return 1
if progress:
sys.stdout.write(self.progress_bar(len(srcs), i + 1, 'scanning...'))
sys.stdout.flush()
if progress:
sys.stdout.write(self.progress_bar(len(copy_fs_paths), 0, self.get_verb()))
sys.stdout.flush()
if self.options.threads > 1:
copy_fs_dirs = [r for r in copy_fs_paths if r[0] == self.DIR]
copy_fs_paths = [r for r in copy_fs_paths if r[0] == self.FILE]
for path_type, fs, path, dest_path in copy_fs_dirs:
dst_fs.makedir(path, allow_recreate=True, recursive=True)
self.lock = threading.RLock()
self.total_files = len(copy_fs_paths)
self.done_files = 0
file_queue = queue.Queue()
threads = [FileOpThread(self.get_action(),
'T%i' % i,
dst_fs,
file_queue,
self.on_done,
self.on_error)
for i in xrange(options.threads)]
for thread in threads:
thread.start()
self.action_errors = []
complete = False
try:
enqueue = file_queue.put
for resource in copy_fs_paths:
enqueue(resource)
while not file_queue.empty():
time.sleep(0)
if self.any_error():
raise SystemExit
# Can't use queue.join here, or KeyboardInterrupt will not be
# caught until the queue is finished
#file_queue.join()
except KeyboardInterrupt:
options.progress = False
self.output("\nCancelling...\n")
except SystemExit:
options.progress = False
finally:
sys.stdout.flush()
for thread in threads:
thread.finish_event.set()
for thread in threads:
thread.join()
complete = True
if not self.any_error():
self.post_actions()
dst_fs.close()
if self.action_errors:
for error in self.action_errors:
self.error(self.wrap_error(unicode(error)) + '\n')
sys.stdout.flush()
else:
if complete and options.progress:
sys.stdout.write(self.progress_bar(self.total_files, self.done_files, ''))
sys.stdout.write('\n')
sys.stdout.flush()
