def copy(source, dest, swift_retry_options=None):
"""Copies a source file to a destination file.
Note that this utility can be called from either OBS, posix, or
windows paths created with ``stor.Path``.
Args:
source (path|str): The source directory to copy from
dest (path|str): The destination file or directory.
swift_retry_options (dict): Optional retry arguments to use for swift
upload or download. View the
`swift module-level documentation <swiftretry>` for more
information on retry arguments
Examples:
Copying a swift file to a local path behaves as follows::
>>> import stor
>>> swift_p = 'swift://tenant/container/dir/file.txt'
>>> # file.txt will be copied to other_dir/other_file.txt
>>> stor.copy(swift_p, 'other_dir/other_file.txt')
Copying from a local path to swift behaves as follows::
>>> from stor import Path
>>> local_p = Path('my/local/file.txt')
>>> # File will be uploaded to swift://tenant/container/dir/my_file.txt
>>> local_p.copy('swift://tenant/container/dir/')
Because of the ambiguity in whether a remote target is a file or directory, copy()
will error on ambiguous paths.
>>> local_p.copy('swift://tenant/container/dir')
Traceback (most recent call last):
...
ValueError: OBS destination must be file with extension or directory with slash
"""
from stor import Path
from stor.obs import OBSUploadObject
source = Path(source)
dest = Path(dest)
swift_retry_options = swift_retry_options or {}
if is_obs_path(source) and is_obs_path(dest):
raise ValueError('cannot copy one OBS path to another OBS path')
if is_obs_path(dest) and dest.is_ambiguous():
raise ValueError('OBS destination must be file with extension or directory with slash')
if is_filesystem_path(dest):
dest.parent.makedirs_p()
if is_obs_path(source):
dest_file = dest if not dest.isdir() else dest / source.name
source.download_object(dest_file, **swift_retry_options)
else:
shutil.copy(source, dest)
else:
dest_file = dest if not dest.endswith('/') else dest / source.name
if is_swift_path(dest) and not dest_file.parent.container:
raise ValueError((
'cannot copy to tenant "%s" and file '
'"%s"' % (dest_file.parent, dest_file.name)
))
dest_obj_name = Path(dest_file.parent.resource or '') / dest_file.name
upload_obj = OBSUploadObject(source, dest_obj_name)
dest_file.parent.upload([upload_obj],
**swift_retry_options)
def copytree(source,
dest,
copy_cmd=None,
use_manifest=False,
headers=None,
condition=None,
**kwargs):
"""Copies a source directory to a destination directory. Assumes that
paths are capable of being copied to/from.
Note that this function uses shutil.copytree by default, meaning
that a posix or windows destination must not exist beforehand.
For example, assume the following file hierarchy::
a/
- b/
- - 1.txt
Doing a copytree from ``a`` to a new posix destination of ``c`` is
performed with::
Path('a').copytree('c')
The end result for c looks like::
c/
- b/
- - 1.txt
Note that the user can override which copy command is used for posix
copies, and it is up to them to ensure that their code abides by the
semantics of the provided copy command. This function has been tested
in production using the default command of ``cp -r`` and using ``mcp -r``.
Using OBS source and destinations work in a similar manner. Assume
the destination is a swift path and we upload the same ``a`` folder::
Path('a').copytree('swift://tenant/container/folder')
The end swift result will have one object::
Path('swift://tenant/container/folder/b/1.txt')
Similarly one can do::
Path('swift://tenant/container/folder/').copytree('c')
The end result for c looks the same as the above posix example::
c/
- b/
- - 1.txt
Args:
source (path|str): The source directory to copy from
dest (path|str): The directory to copy to. Must not exist if
its a posix directory
copy_cmd (str): If copying to / from posix or windows, this command is
used instead of shutil.copytree
use_manifest (bool, default False): See `SwiftPath.upload` and
`SwiftPath.download`.
condition (function(results) -> bool): See `SwiftPath.upload` and
`SwiftPath.download`.
headers (List[str]): See `SwiftPath.upload`.
Raises:
ValueError: if two OBS paths are specified
OSError: if destination is a posix path and it already exists
"""
from stor import Path
source = Path(source)
dest = Path(dest)
if is_dx_path(source) and is_dx_path(dest):
return source.copytree(dest, **kwargs)
if is_obs_path(source) and is_obs_path(dest):
raise ValueError('cannot copy one OBS path to another OBS path')
from stor.windows import WindowsPath
if is_obs_path(source) and isinstance(dest, WindowsPath):
raise ValueError('OBS copytree to windows is not supported')
if is_filesystem_path(dest):
dest.expand().abspath().parent.makedirs_p()
if is_obs_path(source):
source.download(dest,
use_manifest=use_manifest,
condition=condition,
**kwargs)
else:
if copy_cmd:
copy_cmd = shlex.split(copy_cmd)
copy_cmd.extend([
str(source.abspath().expand()),
str(dest.abspath().expand())
])
logger.info('performing copy with command - %s', copy_cmd)
check_call(copy_cmd)
else:
shutil.copytree(source, dest)
else:
if is_dx_path(dest) and (dest.isdir() or dest.endswith('/')):
dest = dest / remove_trailing_slash(source).name
if dest.isdir():
raise exceptions.TargetExistsError(
'Destination path ({}) already exists, will not cause '
'duplicate folders to exist. Remove the original first'.
format(dest))
with source:
dest.upload(['.'],
use_manifest=use_manifest,
headers=headers,
condition=condition,
**kwargs)
