def test_copy_into(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: file0 = paths['file'] newdir = paths['new_dir'] newdir.mkdir() copy(file0, newdir) assert (newdir / file0.name).exists() newdir.rmdir(recursive=True)
def _stage_api_files(self, local_project_dir: cerulean.Path, remote_project_dir: cerulean.Path) -> None: cerulean.copy(local_project_dir / 'version', remote_project_dir / 'version', overwrite='always') local_dir = local_project_dir / 'files' if not local_dir.exists(): self._logger.debug('API files at {} not found, not' ' staging'.format(local_dir)) return remote_dir = remote_project_dir / 'files' self._logger.debug('Staging API part to {} from {}'.format( remote_dir, local_dir)) try_count = 0 succeeded = False while not succeeded and try_count < 10: try: cerulean.copy(local_dir, remote_dir, overwrite='always', copy_into=False, copy_permissions=True) succeeded = True except SSHException as e: self._logger.info('Connection error: {}'.format(e.args[0])) try_count += 1 self._logger.info('Try {} of 10 failed'.format(try_count))
def test_copy_callback(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: def dummy_data() -> Generator[bytes, None, None]: for i in range(20): yield bytes(1024 * 1024) newdir = paths['new_dir'] test_source = newdir / 'source' test_target = newdir / 'target' newdir.mkdir() try: test_source.streaming_write(dummy_data()) num_calls = 0 def callback(count: int, total: int) -> None: nonlocal num_calls num_calls += 1 #time.sleep(2) copy(test_source, test_target, callback=callback) assert num_calls >= 2 finally: newdir.rmdir(recursive=True)
def test_copy_dir_onto_nonexistent(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: dir0 = paths['dir'] newdir = paths['new_dir'] copy(dir0, newdir, overwrite='always', copy_into=False) assert newdir.is_dir() newdir.rmdir(recursive=True)
def publish_job_output(self, job_id: str, output_files: List[File]) -> None: """Write output files to the local output dir for this job. Uses the .output_files property of the job to get data, and updates its .output property with URLs pointing to the newly published files, then sets .output_files to None. Args: job_id: The id of the job whose output to publish. output_files: List of output files to publish. """ self._logger.debug("Publishing output for job " + job_id) job_dir = self._basedir / 'output' / job_id with self._job_store: job = self._job_store.get_job(job_id) if output_files != []: output = json.loads(job.remote_output) self.create_output_dir(job_id) for outf in output_files: out_file = job_dir / outf.location cerulean.copy(cast(Path, outf.source), out_file) output[outf.name]['location'] = self._to_external_url( 'output/' + job_id + '/' + outf.location) output[outf.name]['path'] = str(out_file) job.local_output = json.dumps(output)
def test_copy_file_onto_dir(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: file1 = paths['file'] newdir = paths['new_dir'] newdir.mkdir() copy(file1, newdir, overwrite='always', copy_into=False) assert newdir.is_file() newdir.unlink()
def test_copy_dir_single_fs(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: dir1 = paths['dir'] new_dir = paths['new_dir'] assert not new_dir.exists() copy(dir1, new_dir) assert_dir_copied_correctly(new_dir, filesystem, filesystem) new_dir.rmdir(recursive=True)
def test_copy_dir_single_fs2(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: dir1 = paths['dir'] new_dir = paths['new_dir'] assert not new_dir.exists() copy(dir1, new_dir, overwrite='raise') assert_dir_copied_correctly(new_dir, filesystem, filesystem) with pytest.raises(FileExistsError): copy(dir1, new_dir, overwrite='raise', copy_into=False) new_dir.rmdir(recursive=True)
def test_copy_dir_single_fs3(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: dir1 = paths['dir'] new_dir = paths['new_dir'] assert not new_dir.exists() copy(dir1, new_dir) assert_dir_copied_correctly(new_dir, filesystem, filesystem) (new_dir / 'file0').unlink() copy(dir1, new_dir, overwrite='always', copy_into=False) assert (new_dir / 'file0').exists() new_dir.rmdir(recursive=True) copy(dir1, new_dir, overwrite='always', copy_into=False) assert_dir_copied_correctly(new_dir, filesystem, filesystem) new_dir.rmdir(recursive=True) new_dir.mkdir() copy(dir1, new_dir, overwrite='always', copy_into=True) assert (new_dir / 'links' / 'file0').exists() new_dir.rmdir(recursive=True)
def test_copy_symlink_single_fs(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: if filesystem._supports('symlinks'): link = paths['multi_link'] new_file = paths['new_file'] assert not new_file.exists() copy(link, new_file) assert new_file.exists() assert new_file.is_file() assert new_file.size() == 12 new_file.unlink()
def test_copy_callback_abort(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: test_source = paths['file'] test_target = paths['new_file'] try: def callback(count: int, total: int) -> None: raise RuntimeError() with pytest.raises(RuntimeError): copy(test_source, test_target, callback=callback) assert not test_target.exists() or test_target.size() == 0 finally: if test_target.exists(): test_target.unlink()
def test_no_copy_file_permissions(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: if not filesystem._supports('permissions'): return file1 = paths['executable'] file2 = paths['private'] new_file = paths['new_file'] copy(file1, new_file, copy_permissions=False) assert not new_file.has_permission(Permission.OWNER_EXECUTE) new_file.unlink() new_file.touch() new_file.chmod(0o660) copy(file1, new_file, overwrite='always', copy_permissions=False) assert new_file.has_permission(Permission.OWNER_READ) assert new_file.has_permission(Permission.OWNER_WRITE) assert not new_file.has_permission(Permission.OWNER_EXECUTE) assert not new_file.has_permission(Permission.GROUP_WRITE) assert new_file.has_permission(Permission.OTHERS_READ) new_file.unlink() copy(file2, new_file, overwrite='always', copy_permissions=False) for permission in Permission: assert file2.has_permission(permission) == (permission in [ Permission.OWNER_READ, Permission.OWNER_WRITE ]) new_file.unlink()
def _stage_install_script( self, local_project_dir: cerulean.Path, remote_project_dir: cerulean.Path) -> Optional[cerulean.Path]: local_path = local_project_dir / 'install.sh' if not local_path.exists(): self._logger.debug('API install script not found at {}, not' ' staging'.format(local_path)) return None remote_path = remote_project_dir / 'install.sh' self._logger.debug('Staging API install script to {} from {}'.format( remote_path, local_path)) cerulean.copy(local_path, remote_path, overwrite='always', copy_into=False) while not remote_path.exists(): time.sleep(0.05) remote_path.chmod(0o700) return remote_path
def _stage_input_file(self, count: int, job_id: str, input_file: File, input_desc: Dict[str, Any]) -> int: """Stage an input file. Copies the file to the remote resource. Uses count to create unique file names, returns the new count \ (i.e. the next available number). Args: count: The next available unique count job_id: The job id to stage for input_file: The input file to stage input_desc: The input description whose location \ (and secondaryFiles) to update. Returns: The updated count """ staged_name = _create_input_filename( str(count).zfill(2), input_file.location) count += 1 with self._job_store: job = self._job_store.get_job(job_id) job.info('Staging input file {}'.format(input_file.location)) target_path = self._abs_path(job_id, 'work/{}'.format(staged_name)) cerulean.copy(cast(Path, input_file.source), target_path) input_desc['location'] = str( self._abs_path(job_id, 'work/' + staged_name)) for i, secondary_file in enumerate(input_file.secondary_files): sec_input_desc = input_desc['secondaryFiles'][i] count = self._stage_input_file(count, job_id, secondary_file, sec_input_desc) return count
def test_copy_file_single_fs(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: file1 = paths['file'] new_file = paths['new_file'] assert not new_file.exists() copy(file1, new_file) assert new_file.exists() assert new_file.is_file() new_file.unlink() copy(file1, new_file, overwrite='raise') assert new_file.exists() new_file.unlink() copy(file1, new_file, overwrite='always') assert new_file.exists() new_file.unlink() other_file = paths['other_file'] copy(file1, other_file) assert other_file.size() == 0 with pytest.raises(FileExistsError): copy(file1, other_file, overwrite='raise') copy(file1, other_file, overwrite='always') assert other_file.size() == 12 other_file.unlink() other_file.touch() with pytest.raises(FileNotFoundError): copy(paths['root'] / 'doesnotexist', new_file)
def test_copy_file_args(filesystem: FileSystemImpl, paths: Dict[str, Path]) -> None: file0 = paths['file'] with pytest.raises(ValueError): copy(file0, file0, overwrite='nonexistentoption')
def test_copy_file_cross_fs(filesystem: FileSystemImpl, filesystem2: FileSystemImpl, paths: Dict[str, Path]) -> None: file1 = paths['file'] new_file = filesystem2 / str(paths['new_file']) assert not new_file.exists() copy(file1, new_file) assert new_file.exists() assert new_file.is_file() new_file.unlink() copy(file1, new_file, overwrite='raise') assert new_file.exists() new_file.unlink() copy(file1, new_file, overwrite='always') assert new_file.exists() new_file.unlink() other_file = paths['other_file'] copy(file1, other_file) assert other_file.size() == 0 with pytest.raises(FileExistsError): copy(file1, other_file, overwrite='raise') copy(file1, other_file, overwrite='always') assert other_file.size() == 12 other_file.unlink() other_file.touch()