def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
It uses windows `xcopy` method to do so, making advantage of
server-side copy where available. Different other methods can be used
as robocopy is probably faster, but robocopy doesn't support renaming
destination file when copying just one file :( Shame on you Microsoft.
"""
from subprocess import call
try:
# Python 3.3+
from subprocess import DEVNULL
except ImportError:
# Backwards compatibility
DEVNULL = open(os.devnull, 'w')
# from pathlib import Path, PureWindowsPath
if shutil._samefile(src, dst):
# Get shutil.SameFileError if available (Python 3.4+)
# else fall back to original behavior using shutil.Error
SameFileError = getattr(shutil, "SameFileError", shutil.Error)
raise SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
# call(["xcopy", src, dst], stdin=DEVNULL, stdout=DEVNULL)
cmd = [
"echo", "f", "|", "xcopy", "/y", "/h", "/r",
src.replace('/', '\\'),
dst.replace('/', '\\')
]
# cmd = ["copy", "/B", "/Y",
# src.replace('/', '\\'), dst.replace('/', '\\')]
call(cmd, stdin=DEVNULL, stdout=DEVNULL, shell=True)
# call(["robocopy",
# os.path.dirname(src),
# os.path.dirname(dst),
# os.path.basename(src),
# "/njh", "/njs", "/ndl", "/nc", "/ns", "/nfl"])
# os.rename(os.path.join(
# os.path.dirname(dst), os.path.basename(src)),
# dst)
# call(["copy", src, dst, "/B", "/Y"])
return dst
def copyfile(self, src, dst, *, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
size = os.stat(src).st_size
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
self.copyfileobj(fsrc, fdst, callback=self.draw_copy_progress, total=size)
return dst
def shutil_copyfile(src, dst):
"""Copy data from src to dst"""
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
elif not os.path.exists(src) or os.path.isdir(src):
return
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
try:
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
except NameError:
raise shutil.Error("`%s` is a named pipe" % fn)
BUFFER_SIZE = 128 * 1024
try:
with open(src, "rb") as fin, open(dst, "wb") as fout:
for x in iter(lambda: fin.read(BUFFER_SIZE), ""):
fout.write(x)
except Exception as e:
raise
def copy_file(src, dst, buffer_size=10485760, preserve_file_date=True):
"""
Copies a file to a new location.
Much faster performance than Apache Commons due to use of larger buffer.
:param src: Source file path
:param dst: Destination file path
:param buffer_size: Buffer size to use during copy
:param preserve_file_date: Preserve the original file date
"""
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if buffer_size == 0:
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`{0}` and `{1}` are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError: # File most likely does not exist
pass
else: # XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError(
"`{}` is a named pipe".format(fn))
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
if preserve_file_date:
shutil.copystat(src, dst)
def copyfile(src, dst, *, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
if _sys == 'Linux':
subprocess.check_call(['cp', str(src), str(dst)])
else:
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, length=16 * 1024 * 1024)
return dst
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError(
"{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
# Try to use sendfile if available for performance
if not _copyfile_sendfile(fsrc, fdst):
# sendfile is not available or failed, fallback to copyfileobj
shutil.copyfileobj(fsrc, fdst)
return dst
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
NOTE: this is a copy of shutil.copyfile from python 3.5, modified to be compatible
with python2.7, with the exception of the buffer size
used in copying the file contents.
"""
# noinspection PyUnresolvedReferences,PyProtectedMember
if shutil._samefile(src, dst):
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, length=COPY_BUFFSIZE)
return dst
def copy_file(src, dst, buffer_size=10485760):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param src: Source File
@param dst: Destination File (not file path)
@param buffer_size: Buffer size to use during copy
@param perserveFileDate: Preserve the original file date
'''
dstParent, dstFileName = os.path.split(dst)
if not os.path.exists(dstParent):
os.makedirs(dstParent)
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if (buffer_size == 0):
buffer_size = 1024
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
pass # File most likely does not exist
else:
# TODO: What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
fsrc = open(src, 'rb')
fdst = open(dst, 'wb')
shutil.copyfileobj(fsrc, fdst, buffer_size)
shutil.copystat(src, dst)
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
It uses windows native CopyFileW method to do so, making advantage of
server-side copy where available.
"""
if shutil._samefile(src, dst):
# Get shutil.SameFileError if available (Python 3.4+)
# else fall back to original behavior using shutil.Error
SameFileError = getattr(shutil, "SameFileError", shutil.Error)
raise SameFileError(
"{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
kernel32 = ctypes.WinDLL('kernel32',
use_last_error=True, use_errno=True)
copyfile = kernel32.CopyFile2
copyfile.argtypes = (ctypes.c_wchar_p,
ctypes.c_wchar_p,
ctypes.c_void_p)
copyfile.restype = ctypes.HRESULT
source_file = os.path.normpath(src)
dest_file = os.path.normpath(dst)
if source_file.startswith('\\\\'):
source_file = 'UNC\\' + source_file[2:]
if dest_file.startswith('\\\\'):
dest_file = 'UNC\\' + dest_file[2:]
ret = copyfile('\\\\?\\' + source_file,
'\\\\?\\' + dest_file, None)
if ret != 0:
error = ctypes.get_last_error()
if error == 0:
return dst
# 997 is ERROR_IO_PENDING. Why it is poping here with
# CopyFileW is beyond me, but assume we can easily
# ignore it as it is copying nevertheless
if error == 997:
return dst
raise IOError(
"File {!r} copy failed, error: {}".format(src, error))
return dst
def copy_file(src, dst, buffer_size=10485760, perserveFileDate=None):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param src: Source File
@param dst: Destination File (not file path)
@param buffer_size: Buffer size to use during copy
@param perserveFileDate: Preserve the original file date
'''
# Check to make sure destination directory exists. If it doesn't create the directory
dstParent, dstFileName = os.path.split(dst)
if (not (os.path.exists(dstParent))):
os.makedirs(dstParent)
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if (buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
src_fh = os.open(src, os.O_RDONLY | os.O_SYNC)
dst_fh = os.open(dst,
os.O_CREAT | os.O_SYNC | os.O_TRUNC | os.O_WRONLY)
if src_fh != None and dst_fh != None:
while True:
buffer = os.read(src_fh, lock.BUFFER_SIZE)
if buffer == '':
break
os.write(dst_fh, buffer)
if src_fh:
os.close(src_fh)
if dst_fh:
os.close(dst_fh)
# shutil.copyfileobj(fsrc, fdst, buffer_size)
#if lock.debug: lock.info("Lock","\n".join(f.readlines()))
if (perserveFileDate):
shutil.copystat(src, dst)
def copyFile(src, dst, buffer_size=10485760, perserveFileDate=True):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param src: Source File
@param dst: Destination File (not file path)
@param buffer_size: Buffer size to use during copy
@param perserveFileDate: Preserve the original file date
'''
# Check to make sure destination directory exists. If it doesn't create the directory
dstParent, dstFileName = os.path.split(dst)
if(not(os.path.exists(dstParent))):
os.makedirs.(dstParent)
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if(buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
# with open(src, 'rb') as fsrc:
# with open(dst, 'wb') as fdst:
# shutil.copyfileobj(fsrc, fdst, buffer_size)
fsrc = open(src, 'rb')
try:
fdst = open(dst, 'wb')
try:
shutil.copyfileobj(fsrc, fdst, buffer_size)
finally:
fdst.close()
finally:
fsrc.close()
if(perserveFileDate):
shutil.copystat(src, dst)
def copy2_fixed(src, dest):
# type: (str, str) -> None
"""Wrap shutil.copy2() but map errors copying socket files to
SpecialFileError as expected.
See also https://bugs.python.org/issue37700.
"""
try:
shutil.copy2(src, dest)
except OSError:
for f in [src, dest]:
try:
is_socket_file = is_socket(f)
except OSError:
# An error has already occurred. Another error here is not
# a problem and we can ignore it.
pass
else:
if is_socket_file:
raise shutil.SpecialFileError(
"`{f}` is a socket".format(**locals()))
raise
def fast_copyfile(src, dst, buffer_size=1024 * 1024):
"""
Copy data from src to dst - reimplemented with a buffer size
Note that this function is simply a copy of the function from the official python shutils.py file, but
with an increased (configurable) buffer size fed into copyfileobj instead of the original 16kb one
"""
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
def copyfile(src, dst, *, follow_symlinks=True):
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
pass
else:
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
size = os.stat(src).st_size
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst, callback=copy_progress, total=size)
return dst
def copyFile(src, dst, buffer_size=10485760, perserveFileDate=True):
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if(buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
return
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
pass
else:
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
if(perserveFileDate):
shutil.copystat(src, dst)
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError(
"{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError as e:
# File most likely does not exist
debug(">>> {} doesn't exists [ {} ]".format(fn, e))
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
debug(">>> creating symlink ...")
os.symlink(os.readlink(src), dst)
else:
fs_src_type = FilesystemInfo().filesystem(src.encode('utf-8'))
fs_dst_type = FilesystemInfo().filesystem(
os.path.dirname(dst.encode('utf-8')))
supported_fs = ['CIFS', 'SMB2']
debug(">>> Source FS: {}".format(fs_src_type))
debug(">>> Destination FS: {}".format(fs_dst_type))
if fs_src_type in supported_fs and fs_dst_type in supported_fs:
fsrc = os.open(src, os.O_RDONLY)
fdst = os.open(dst, os.O_WRONLY | os.O_CREAT)
CIFS_IOCTL_MAGIC = 0xCF
CIFS_IOC_COPYCHUNK_FILE = IOW(CIFS_IOCTL_MAGIC, 3, c_int)
# try copy file with COW support on Linux. If fail, fallback
# to sendfile and if this is not available too, fallback
# copyfileobj.
ret = ioctl(fdst, CIFS_IOC_COPYCHUNK_FILE, fsrc)
os.close(fsrc)
os.close(fdst)
if ret != 0:
debug("!!! failed {}".format(ret))
os.close(fsrc)
os.close(fdst)
# Try to use sendfile if available for performance
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
if not _copyfile_sendfile(fsrc, fdst):
debug("!!! failed sendfile")
# sendfile is not available or failed, fallback
# to copyfileobj
shutil.copyfileobj(fsrc, fdst)
else:
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
if not _copyfile_sendfile(fsrc, fdst):
# sendfile is not available or failed, fallback
# to copyfileobj
shutil.copyfileobj(src, dst)
return dst
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
It uses windows native ``CopyFile2`` method to do so, making advantage
of server-side copy where available. If this method is not available
it will fallback to ``CopyFileW`` (on Windows 7 and older).
Args:
src (str): Source file.
dst (str): Destination file.
follow_symlinks (bool): If ``follow_symlinks`` is not set and
``src`` is a symbolic link, a new symlink will be created
instead of copying the file it points to.
Returns:
str: Destination on success
Raises:
shutil.SpecialFileError: when source/destination is invalid.
shutil.SameFileError: if ``src`` and ``dst`` are same.
OSError: if file no exist
IOError: if copying failed on windows API level.
"""
if shutil._samefile(src, dst):
# Get shutil.SameFileError if available (Python 3.4+)
# else fall back to original behavior using shutil.Error
SameFileError = getattr(shutil, "SameFileError", shutil.Error)
raise SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
kernel32 = ctypes.WinDLL('kernel32',
use_last_error=True,
use_errno=True)
try:
copyfile = kernel32.CopyFile2
except AttributeError:
# on windows 7 and older
copyfile = kernel32.CopyFileW
copyfile.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
ctypes.c_void_p)
copyfile.restype = ctypes.HRESULT
source_file = os.path.abspath(os.path.normpath(src))
dest_file = os.path.abspath(os.path.normpath(dst))
if source_file.startswith('\\\\'):
source_file = 'UNC\\' + source_file[2:]
if dest_file.startswith('\\\\'):
dest_file = 'UNC\\' + dest_file[2:]
ret = copyfile('\\\\?\\' + source_file, '\\\\?\\' + dest_file,
None)
if ret == 0:
error = ctypes.get_last_error()
if error == 0:
return dst
# 997 is ERROR_IO_PENDING. Why it is poping here with
# CopyFileW is beyond me, but assume we can easily
# ignore it as it is copying nevertheless
if error == 997:
return dst
raise IOError("File {!r} copy failed, error: {}".format(
src, ctypes.FormatError(error)))
return dst
