def test_url_join(): assert url_join('/path/to', 'file') == '/path/to/file' assert url_join('/path/to/', 'file') == '/path/to/file' assert url_join('/path/to/', '/absolute/file') == '/absolute/file' assert url_join('https://hail.is/path/to', 'file') == 'https://hail.is/path/to/file' assert url_join('https://hail.is/path/to/', 'file') == 'https://hail.is/path/to/file' assert url_join('https://hail.is/path/to/', '/absolute/file') == 'https://hail.is/absolute/file'
async def copy_source(srcentry): srcfile = srcentry.url_maybe_trailing_slash() assert srcfile.startswith(src) # skip files with empty names if srcfile.endswith('/'): return relsrcfile = srcfile[len(src):] assert not relsrcfile.startswith('/') await self._copy_file_multi_part(sema, source_report, srcfile, await srcentry.status(), url_join(full_dest, relsrcfile), return_exceptions)
async def _full_dest(self): if self.dest_type_task: dest_type = await self.dest_type_task else: dest_type = None if (self.treat_dest_as == Transfer.DEST_DIR or (self.treat_dest_as == Transfer.INFER_DEST and dest_type == AsyncFS.DIR)): # We know dest is a dir, but we're copying to # dest/basename(src), and we don't know its type. return url_join(self.dest, url_basename(self.src.rstrip('/'))), None if (self.treat_dest_as == Transfer.DEST_IS_TARGET and self.dest.endswith('/')): dest_type = AsyncFS.DIR return self.dest, dest_type
async def _full_dest(self): dest_type = await self.dest_type_task if (self.treat_dest_as == Transfer.TARGET_DIR or self.dest.endswith('/') or (self.treat_dest_as == Transfer.INFER_TARGET and dest_type == AsyncFS.DIR)): if dest_type is None: raise FileNotFoundError(self.dest) if dest_type == AsyncFS.FILE: raise NotADirectoryError(self.dest) assert dest_type == AsyncFS.DIR # We know dest is a dir, but we're copying to # dest/basename(src), and we don't know its type. return url_join(self.dest, url_basename(self.src.rstrip('/'))), None assert not self.dest.endswith('/') return self.dest, dest_type
async def copy_as_dir(self, worker_pool: AsyncWorkerPool, source_report: SourceReport): src = self.src if not src.endswith('/'): src = src + '/' try: srcentries = await self.router_fs.listfiles(src, recursive=True) except (NotADirectoryError, FileNotFoundError): self.src_is_dir = False await self.release_barrier() return self.src_is_dir = True await self.release_barrier_and_wait() if self.src_is_file: raise FileAndDirectoryError(self.src) source_report._source_type = AsyncFS.DIR full_dest, full_dest_type = await self._full_dest() if full_dest_type == AsyncFS.FILE: raise NotADirectoryError(full_dest) async with WaitableSharedPool(worker_pool) as pool: async for srcentry in srcentries: srcfile = srcentry.url_maybe_trailing_slash() assert srcfile.startswith(src) # skip files with empty names if srcfile.endswith('/'): continue relsrcfile = srcfile[len(src):] assert not relsrcfile.startswith('/') await pool.call(self._copy_file, source_report, srcfile, url_join(full_dest, relsrcfile))