def test_copy_into(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None:
file0 = paths['file']
newdir = paths['new_dir']
newdir.mkdir()
copy(file0, newdir)
assert (newdir / file0.name).exists()
newdir.rmdir(recursive=True)
def _stage_api_files(self, local_project_dir: cerulean.Path,
remote_project_dir: cerulean.Path) -> None:
cerulean.copy(local_project_dir / 'version',
remote_project_dir / 'version',
overwrite='always')
local_dir = local_project_dir / 'files'
if not local_dir.exists():
self._logger.debug('API files at {} not found, not'
' staging'.format(local_dir))
return
remote_dir = remote_project_dir / 'files'
self._logger.debug('Staging API part to {} from {}'.format(
remote_dir, local_dir))
try_count = 0
succeeded = False
while not succeeded and try_count < 10:
try:
cerulean.copy(local_dir,
remote_dir,
overwrite='always',
copy_into=False,
copy_permissions=True)
succeeded = True
except SSHException as e:
self._logger.info('Connection error: {}'.format(e.args[0]))
try_count += 1
self._logger.info('Try {} of 10 failed'.format(try_count))
def test_copy_callback(filesystem: FileSystemImpl, paths: Dict[str,
Path]) -> None:
def dummy_data() -> Generator[bytes, None, None]:
for i in range(20):
yield bytes(1024 * 1024)
newdir = paths['new_dir']
test_source = newdir / 'source'
test_target = newdir / 'target'
newdir.mkdir()
try:
test_source.streaming_write(dummy_data())
num_calls = 0
def callback(count: int, total: int) -> None:
nonlocal num_calls
num_calls += 1
#time.sleep(2)
copy(test_source, test_target, callback=callback)
assert num_calls >= 2
finally:
newdir.rmdir(recursive=True)
def test_copy_dir_onto_nonexistent(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
dir0 = paths['dir']
newdir = paths['new_dir']
copy(dir0, newdir, overwrite='always', copy_into=False)
assert newdir.is_dir()
newdir.rmdir(recursive=True)
def publish_job_output(self, job_id: str,
output_files: List[File]) -> None:
"""Write output files to the local output dir for this job.
Uses the .output_files property of the job to get data, and
updates its .output property with URLs pointing to the newly
published files, then sets .output_files to None.
Args:
job_id: The id of the job whose output to publish.
output_files: List of output files to publish.
"""
self._logger.debug("Publishing output for job " + job_id)
job_dir = self._basedir / 'output' / job_id
with self._job_store:
job = self._job_store.get_job(job_id)
if output_files != []:
output = json.loads(job.remote_output)
self.create_output_dir(job_id)
for outf in output_files:
out_file = job_dir / outf.location
cerulean.copy(cast(Path, outf.source), out_file)
output[outf.name]['location'] = self._to_external_url(
'output/' + job_id + '/' + outf.location)
output[outf.name]['path'] = str(out_file)
job.local_output = json.dumps(output)
def test_copy_file_onto_dir(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
file1 = paths['file']
newdir = paths['new_dir']
newdir.mkdir()
copy(file1, newdir, overwrite='always', copy_into=False)
assert newdir.is_file()
newdir.unlink()
def test_copy_dir_single_fs(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
dir1 = paths['dir']
new_dir = paths['new_dir']
assert not new_dir.exists()
copy(dir1, new_dir)
assert_dir_copied_correctly(new_dir, filesystem, filesystem)
new_dir.rmdir(recursive=True)
def test_copy_dir_single_fs2(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
dir1 = paths['dir']
new_dir = paths['new_dir']
assert not new_dir.exists()
copy(dir1, new_dir, overwrite='raise')
assert_dir_copied_correctly(new_dir, filesystem, filesystem)
with pytest.raises(FileExistsError):
copy(dir1, new_dir, overwrite='raise', copy_into=False)
new_dir.rmdir(recursive=True)
def test_copy_dir_single_fs3(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
dir1 = paths['dir']
new_dir = paths['new_dir']
assert not new_dir.exists()
copy(dir1, new_dir)
assert_dir_copied_correctly(new_dir, filesystem, filesystem)
(new_dir / 'file0').unlink()
copy(dir1, new_dir, overwrite='always', copy_into=False)
assert (new_dir / 'file0').exists()
new_dir.rmdir(recursive=True)
copy(dir1, new_dir, overwrite='always', copy_into=False)
assert_dir_copied_correctly(new_dir, filesystem, filesystem)
new_dir.rmdir(recursive=True)
new_dir.mkdir()
copy(dir1, new_dir, overwrite='always', copy_into=True)
assert (new_dir / 'links' / 'file0').exists()
new_dir.rmdir(recursive=True)
def test_copy_symlink_single_fs(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
if filesystem._supports('symlinks'):
link = paths['multi_link']
new_file = paths['new_file']
assert not new_file.exists()
copy(link, new_file)
assert new_file.exists()
assert new_file.is_file()
assert new_file.size() == 12
new_file.unlink()
def test_copy_callback_abort(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
test_source = paths['file']
test_target = paths['new_file']
try:
def callback(count: int, total: int) -> None:
raise RuntimeError()
with pytest.raises(RuntimeError):
copy(test_source, test_target, callback=callback)
assert not test_target.exists() or test_target.size() == 0
finally:
if test_target.exists():
test_target.unlink()
def test_no_copy_file_permissions(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
if not filesystem._supports('permissions'):
return
file1 = paths['executable']
file2 = paths['private']
new_file = paths['new_file']
copy(file1, new_file, copy_permissions=False)
assert not new_file.has_permission(Permission.OWNER_EXECUTE)
new_file.unlink()
new_file.touch()
new_file.chmod(0o660)
copy(file1, new_file, overwrite='always', copy_permissions=False)
assert new_file.has_permission(Permission.OWNER_READ)
assert new_file.has_permission(Permission.OWNER_WRITE)
assert not new_file.has_permission(Permission.OWNER_EXECUTE)
assert not new_file.has_permission(Permission.GROUP_WRITE)
assert new_file.has_permission(Permission.OTHERS_READ)
new_file.unlink()
copy(file2, new_file, overwrite='always', copy_permissions=False)
for permission in Permission:
assert file2.has_permission(permission) == (permission in [
Permission.OWNER_READ, Permission.OWNER_WRITE
])
new_file.unlink()
def _stage_install_script(
self, local_project_dir: cerulean.Path,
remote_project_dir: cerulean.Path) -> Optional[cerulean.Path]:
local_path = local_project_dir / 'install.sh'
if not local_path.exists():
self._logger.debug('API install script not found at {}, not'
' staging'.format(local_path))
return None
remote_path = remote_project_dir / 'install.sh'
self._logger.debug('Staging API install script to {} from {}'.format(
remote_path, local_path))
cerulean.copy(local_path,
remote_path,
overwrite='always',
copy_into=False)
while not remote_path.exists():
time.sleep(0.05)
remote_path.chmod(0o700)
return remote_path
def _stage_input_file(self, count: int, job_id: str, input_file: File,
input_desc: Dict[str, Any]) -> int:
"""Stage an input file. Copies the file to the remote resource.
Uses count to create unique file names, returns the new count \
(i.e. the next available number).
Args:
count: The next available unique count
job_id: The job id to stage for
input_file: The input file to stage
input_desc: The input description whose location \
(and secondaryFiles) to update.
Returns:
The updated count
"""
staged_name = _create_input_filename(
str(count).zfill(2), input_file.location)
count += 1
with self._job_store:
job = self._job_store.get_job(job_id)
job.info('Staging input file {}'.format(input_file.location))
target_path = self._abs_path(job_id, 'work/{}'.format(staged_name))
cerulean.copy(cast(Path, input_file.source), target_path)
input_desc['location'] = str(
self._abs_path(job_id, 'work/' + staged_name))
for i, secondary_file in enumerate(input_file.secondary_files):
sec_input_desc = input_desc['secondaryFiles'][i]
count = self._stage_input_file(count, job_id, secondary_file,
sec_input_desc)
return count
def test_copy_file_single_fs(filesystem: FileSystemImpl,
paths: Dict[str, Path]) -> None:
file1 = paths['file']
new_file = paths['new_file']
assert not new_file.exists()
copy(file1, new_file)
assert new_file.exists()
assert new_file.is_file()
new_file.unlink()
copy(file1, new_file, overwrite='raise')
assert new_file.exists()
new_file.unlink()
copy(file1, new_file, overwrite='always')
assert new_file.exists()
new_file.unlink()
other_file = paths['other_file']
copy(file1, other_file)
assert other_file.size() == 0
with pytest.raises(FileExistsError):
copy(file1, other_file, overwrite='raise')
copy(file1, other_file, overwrite='always')
assert other_file.size() == 12
other_file.unlink()
other_file.touch()
with pytest.raises(FileNotFoundError):
copy(paths['root'] / 'doesnotexist', new_file)
def test_copy_file_args(filesystem: FileSystemImpl, paths: Dict[str,
Path]) -> None:
file0 = paths['file']
with pytest.raises(ValueError):
copy(file0, file0, overwrite='nonexistentoption')
def test_copy_file_cross_fs(filesystem: FileSystemImpl,
filesystem2: FileSystemImpl,
paths: Dict[str, Path]) -> None:
file1 = paths['file']
new_file = filesystem2 / str(paths['new_file'])
assert not new_file.exists()
copy(file1, new_file)
assert new_file.exists()
assert new_file.is_file()
new_file.unlink()
copy(file1, new_file, overwrite='raise')
assert new_file.exists()
new_file.unlink()
copy(file1, new_file, overwrite='always')
assert new_file.exists()
new_file.unlink()
other_file = paths['other_file']
copy(file1, other_file)
assert other_file.size() == 0
with pytest.raises(FileExistsError):
copy(file1, other_file, overwrite='raise')
copy(file1, other_file, overwrite='always')
assert other_file.size() == 12
other_file.unlink()
other_file.touch()
