def test_url_join():
assert url_join('/path/to', 'file') == '/path/to/file'
assert url_join('/path/to/', 'file') == '/path/to/file'
assert url_join('/path/to/', '/absolute/file') == '/absolute/file'
assert url_join('https://hail.is/path/to', 'file') == 'https://hail.is/path/to/file'
assert url_join('https://hail.is/path/to/', 'file') == 'https://hail.is/path/to/file'
assert url_join('https://hail.is/path/to/', '/absolute/file') == 'https://hail.is/absolute/file'
async def copy_source(srcentry):
srcfile = srcentry.url_maybe_trailing_slash()
assert srcfile.startswith(src)
# skip files with empty names
if srcfile.endswith('/'):
return
relsrcfile = srcfile[len(src):]
assert not relsrcfile.startswith('/')
await self._copy_file_multi_part(sema, source_report, srcfile,
await srcentry.status(),
url_join(full_dest, relsrcfile),
return_exceptions)
async def _full_dest(self):
if self.dest_type_task:
dest_type = await self.dest_type_task
else:
dest_type = None
if (self.treat_dest_as == Transfer.DEST_DIR
or (self.treat_dest_as == Transfer.INFER_DEST
and dest_type == AsyncFS.DIR)):
# We know dest is a dir, but we're copying to
# dest/basename(src), and we don't know its type.
return url_join(self.dest, url_basename(self.src.rstrip('/'))), None
if (self.treat_dest_as == Transfer.DEST_IS_TARGET
and self.dest.endswith('/')):
dest_type = AsyncFS.DIR
return self.dest, dest_type
async def _full_dest(self):
dest_type = await self.dest_type_task
if (self.treat_dest_as == Transfer.TARGET_DIR
or self.dest.endswith('/')
or (self.treat_dest_as == Transfer.INFER_TARGET
and dest_type == AsyncFS.DIR)):
if dest_type is None:
raise FileNotFoundError(self.dest)
if dest_type == AsyncFS.FILE:
raise NotADirectoryError(self.dest)
assert dest_type == AsyncFS.DIR
# We know dest is a dir, but we're copying to
# dest/basename(src), and we don't know its type.
return url_join(self.dest,
url_basename(self.src.rstrip('/'))), None
assert not self.dest.endswith('/')
return self.dest, dest_type
async def copy_as_dir(self, worker_pool: AsyncWorkerPool,
source_report: SourceReport):
src = self.src
if not src.endswith('/'):
src = src + '/'
try:
srcentries = await self.router_fs.listfiles(src, recursive=True)
except (NotADirectoryError, FileNotFoundError):
self.src_is_dir = False
await self.release_barrier()
return
self.src_is_dir = True
await self.release_barrier_and_wait()
if self.src_is_file:
raise FileAndDirectoryError(self.src)
source_report._source_type = AsyncFS.DIR
full_dest, full_dest_type = await self._full_dest()
if full_dest_type == AsyncFS.FILE:
raise NotADirectoryError(full_dest)
async with WaitableSharedPool(worker_pool) as pool:
async for srcentry in srcentries:
srcfile = srcentry.url_maybe_trailing_slash()
assert srcfile.startswith(src)
# skip files with empty names
if srcfile.endswith('/'):
continue
relsrcfile = srcfile[len(src):]
assert not relsrcfile.startswith('/')
await pool.call(self._copy_file, source_report, srcfile,
url_join(full_dest, relsrcfile))