def copy_input(job, r):
if isinstance(r, resource.InputResourceFile):
if r not in copied_input_resource_files:
copied_input_resource_files.add(r)
input_scheme = url_scheme(r._input_path)
if input_scheme != '':
transfers_bytes = orjson.dumps([{
"from":
r._input_path,
"to":
r._get_path(tmpdir)
}])
transfers = transfers_bytes.decode('utf-8')
return [
f'python3 -m hailtop.aiotools.copy {shq(requester_pays_project_json)} {shq(transfers)}'
]
absolute_input_path = os.path.realpath(
os.path.expanduser(r._input_path))
dest = r._get_path(os.path.expanduser(tmpdir))
dir = os.path.dirname(dest)
os.makedirs(dir, exist_ok=True)
if job._image is not None: # pylint: disable-msg=W0640
return [f'cp {shq(absolute_input_path)} {shq(dest)}']
return [f'ln -sf {shq(absolute_input_path)} {shq(dest)}']
return []
assert isinstance(
r, (resource.JobResourceFile, resource.PythonResult))
return []
async def test_cloud_rmtree_file_ending_in_slash(
filesystem: Tuple[asyncio.Semaphore, AsyncFS, str]):
sema, fs, base = filesystem
base_scheme = url_scheme(base)
if isinstance(fs, LocalAsyncFS) or base_scheme in ('', 'file'):
return
fname = f'{base}bar/'
async with await fs.create(fname) as f:
await f.write(b'test_rmtree_file_ending_in_slash')
await fs.rmtree(sema, fname)
assert not await fs.isdir(fname)
assert not await fs.isfile(fname)
assert not await fs.exists(fname)
async def test_copy_behavior(copy_test_context, test_spec):
sema, fs, src_base, dest_base = copy_test_context
result = await run_test_spec(sema, fs, test_spec, src_base, dest_base)
try:
expected = test_spec['result']
dest_scheme = url_scheme(dest_base)
if (dest_scheme == 'gs'
and 'files' in result
and expected.get('exception') in ('IsADirectoryError', 'NotADirectoryError')):
return
assert result == expected, (test_spec, result, expected)
except Exception:
print(test_spec)
raise
def test_url_scheme():
assert url_scheme('https://hail.is/path/to') == 'https'
assert url_scheme('/path/to') == ''
def __init__(self, dest_base, expected_type):
scheme = url_scheme(dest_base)
self._object_store = (scheme == 'gs' or scheme == 's3'
or scheme == 'hail-az')
self._expected_type = expected_type
def __init__(self, dest_base, expected_type):
self._gs = url_scheme(dest_base) == 'gs'
self._expected_type = expected_type