def copyfile(src, dst, follow_symlinks=True): """Copy data from src to dst. It uses windows `xcopy` method to do so, making advantage of server-side copy where available. Different other methods can be used as robocopy is probably faster, but robocopy doesn't support renaming destination file when copying just one file :( Shame on you Microsoft. """ from subprocess import call try: # Python 3.3+ from subprocess import DEVNULL except ImportError: # Backwards compatibility DEVNULL = open(os.devnull, 'w') # from pathlib import Path, PureWindowsPath if shutil._samefile(src, dst): # Get shutil.SameFileError if available (Python 3.4+) # else fall back to original behavior using shutil.Error SameFileError = getattr(shutil, "SameFileError", shutil.Error) raise SameFileError("{!r} and {!r} are the same file".format( src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: # call(["xcopy", src, dst], stdin=DEVNULL, stdout=DEVNULL) cmd = [ "echo", "f", "|", "xcopy", "/y", "/h", "/r", src.replace('/', '\\'), dst.replace('/', '\\') ] # cmd = ["copy", "/B", "/Y", # src.replace('/', '\\'), dst.replace('/', '\\')] call(cmd, stdin=DEVNULL, stdout=DEVNULL, shell=True) # call(["robocopy", # os.path.dirname(src), # os.path.dirname(dst), # os.path.basename(src), # "/njh", "/njs", "/ndl", "/nc", "/ns", "/nfl"]) # os.rename(os.path.join( # os.path.dirname(dst), os.path.basename(src)), # dst) # call(["copy", src, dst, "/B", "/Y"]) return dst
def copyfile(self, src, dst, *, follow_symlinks=True): """Copy data from src to dst. If follow_symlinks is not set and src is a symbolic link, a new symlink will be created instead of copying the file it points to. """ if shutil._samefile(src, dst): raise shutil.SameFileError("{!r} and {!r} are the same file".format(src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: size = os.stat(src).st_size with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: self.copyfileobj(fsrc, fdst, callback=self.draw_copy_progress, total=size) return dst
def shutil_copyfile(src, dst): """Copy data from src to dst""" if shutil._samefile(src, dst): raise shutil.Error("`%s` and `%s` are the same file" % (src, dst)) elif not os.path.exists(src) or os.path.isdir(src): return for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): try: raise shutil.SpecialFileError("`%s` is a named pipe" % fn) except NameError: raise shutil.Error("`%s` is a named pipe" % fn) BUFFER_SIZE = 128 * 1024 try: with open(src, "rb") as fin, open(dst, "wb") as fout: for x in iter(lambda: fin.read(BUFFER_SIZE), ""): fout.write(x) except Exception as e: raise
def copy_file(src, dst, buffer_size=10485760, preserve_file_date=True): """ Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer. :param src: Source file path :param dst: Destination file path :param buffer_size: Buffer size to use during copy :param preserve_file_date: Preserve the original file date """ # Optimize the buffer for small files buffer_size = min(buffer_size, os.path.getsize(src)) if buffer_size == 0: buffer_size = 1024 if shutil._samefile(src, dst): raise shutil.Error("`{0}` and `{1}` are the same file".format( src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError( "`{}` is a named pipe".format(fn)) with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: shutil.copyfileobj(fsrc, fdst, buffer_size) if preserve_file_date: shutil.copystat(src, dst)
def copyfile(src, dst, *, follow_symlinks=True): """Copy data from src to dst. If follow_symlinks is not set and src is a symbolic link, a new symlink will be created instead of copying the file it points to. """ if shutil._samefile(src, dst): raise shutil.SameFileError("{!r} and {!r} are the same file".format( src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: if _sys == 'Linux': subprocess.check_call(['cp', str(src), str(dst)]) else: with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: shutil.copyfileobj(fsrc, fdst, length=16 * 1024 * 1024) return dst
def copyfile(src, dst, follow_symlinks=True): """Copy data from src to dst. If follow_symlinks is not set and src is a symbolic link, a new symlink will be created instead of copying the file it points to. """ if shutil._samefile(src, dst): raise shutil.SameFileError( "{!r} and {!r} are the same file".format(src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: # Try to use sendfile if available for performance if not _copyfile_sendfile(fsrc, fdst): # sendfile is not available or failed, fallback to copyfileobj shutil.copyfileobj(fsrc, fdst) return dst
def copyfile(src, dst, follow_symlinks=True): """Copy data from src to dst. If follow_symlinks is not set and src is a symbolic link, a new symlink will be created instead of copying the file it points to. NOTE: this is a copy of shutil.copyfile from python 3.5, modified to be compatible with python2.7, with the exception of the buffer size used in copying the file contents. """ # noinspection PyUnresolvedReferences,PyProtectedMember if shutil._samefile(src, dst): raise SameFileError("{!r} and {!r} are the same file".format(src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: shutil.copyfileobj(fsrc, fdst, length=COPY_BUFFSIZE) return dst
def copy_file(src, dst, buffer_size=10485760): ''' Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer @param src: Source File @param dst: Destination File (not file path) @param buffer_size: Buffer size to use during copy @param perserveFileDate: Preserve the original file date ''' dstParent, dstFileName = os.path.split(dst) if not os.path.exists(dstParent): os.makedirs(dstParent) # Optimize the buffer for small files buffer_size = min(buffer_size, os.path.getsize(src)) if (buffer_size == 0): buffer_size = 1024 for fn in [src, dst]: try: st = os.stat(fn) except OSError: pass # File most likely does not exist else: # TODO: What about other special files? (sockets, devices...) if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) fsrc = open(src, 'rb') fdst = open(dst, 'wb') shutil.copyfileobj(fsrc, fdst, buffer_size) shutil.copystat(src, dst)
def copyfile(src, dst, follow_symlinks=True): """Copy data from src to dst. It uses windows native CopyFileW method to do so, making advantage of server-side copy where available. """ if shutil._samefile(src, dst): # Get shutil.SameFileError if available (Python 3.4+) # else fall back to original behavior using shutil.Error SameFileError = getattr(shutil, "SameFileError", shutil.Error) raise SameFileError( "{!r} and {!r} are the same file".format(src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: kernel32 = ctypes.WinDLL('kernel32', use_last_error=True, use_errno=True) copyfile = kernel32.CopyFile2 copyfile.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p) copyfile.restype = ctypes.HRESULT source_file = os.path.normpath(src) dest_file = os.path.normpath(dst) if source_file.startswith('\\\\'): source_file = 'UNC\\' + source_file[2:] if dest_file.startswith('\\\\'): dest_file = 'UNC\\' + dest_file[2:] ret = copyfile('\\\\?\\' + source_file, '\\\\?\\' + dest_file, None) if ret != 0: error = ctypes.get_last_error() if error == 0: return dst # 997 is ERROR_IO_PENDING. Why it is poping here with # CopyFileW is beyond me, but assume we can easily # ignore it as it is copying nevertheless if error == 997: return dst raise IOError( "File {!r} copy failed, error: {}".format(src, error)) return dst
def copy_file(src, dst, buffer_size=10485760, perserveFileDate=None): ''' Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer @param src: Source File @param dst: Destination File (not file path) @param buffer_size: Buffer size to use during copy @param perserveFileDate: Preserve the original file date ''' # Check to make sure destination directory exists. If it doesn't create the directory dstParent, dstFileName = os.path.split(dst) if (not (os.path.exists(dstParent))): os.makedirs(dstParent) # Optimize the buffer for small files buffer_size = min(buffer_size, os.path.getsize(src)) if (buffer_size == 0): buffer_size = 1024 if shutil._samefile(src, dst): raise shutil.Error("`%s` and `%s` are the same file" % (src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) src_fh = os.open(src, os.O_RDONLY | os.O_SYNC) dst_fh = os.open(dst, os.O_CREAT | os.O_SYNC | os.O_TRUNC | os.O_WRONLY) if src_fh != None and dst_fh != None: while True: buffer = os.read(src_fh, lock.BUFFER_SIZE) if buffer == '': break os.write(dst_fh, buffer) if src_fh: os.close(src_fh) if dst_fh: os.close(dst_fh) # shutil.copyfileobj(fsrc, fdst, buffer_size) #if lock.debug: lock.info("Lock","\n".join(f.readlines())) if (perserveFileDate): shutil.copystat(src, dst)
def copyFile(src, dst, buffer_size=10485760, perserveFileDate=True): ''' Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer @param src: Source File @param dst: Destination File (not file path) @param buffer_size: Buffer size to use during copy @param perserveFileDate: Preserve the original file date ''' # Check to make sure destination directory exists. If it doesn't create the directory dstParent, dstFileName = os.path.split(dst) if(not(os.path.exists(dstParent))): os.makedirs.(dstParent) # Optimize the buffer for small files buffer_size = min(buffer_size, os.path.getsize(src)) if(buffer_size == 0): buffer_size = 1024 if shutil._samefile(src, dst): raise shutil.Error("`%s` and `%s` are the same file" % (src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) # with open(src, 'rb') as fsrc: # with open(dst, 'wb') as fdst: # shutil.copyfileobj(fsrc, fdst, buffer_size) fsrc = open(src, 'rb') try: fdst = open(dst, 'wb') try: shutil.copyfileobj(fsrc, fdst, buffer_size) finally: fdst.close() finally: fsrc.close() if(perserveFileDate): shutil.copystat(src, dst)
def copy2_fixed(src, dest): # type: (str, str) -> None """Wrap shutil.copy2() but map errors copying socket files to SpecialFileError as expected. See also https://bugs.python.org/issue37700. """ try: shutil.copy2(src, dest) except OSError: for f in [src, dest]: try: is_socket_file = is_socket(f) except OSError: # An error has already occurred. Another error here is not # a problem and we can ignore it. pass else: if is_socket_file: raise shutil.SpecialFileError( "`{f}` is a socket".format(**locals())) raise
def fast_copyfile(src, dst, buffer_size=1024 * 1024): """ Copy data from src to dst - reimplemented with a buffer size Note that this function is simply a copy of the function from the official python shutils.py file, but with an increased (configurable) buffer size fed into copyfileobj instead of the original 16kb one """ if shutil._samefile(src, dst): raise shutil.Error("`%s` and `%s` are the same file" % (src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: shutil.copyfileobj(fsrc, fdst, buffer_size)
def copyfile(src, dst, *, follow_symlinks=True): if shutil._samefile(src, dst): raise shutil.SameFileError("{!r} and {!r} are the same file".format( src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: pass else: if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: size = os.stat(src).st_size with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: copyfileobj(fsrc, fdst, callback=copy_progress, total=size) return dst
def copyFile(src, dst, buffer_size=10485760, perserveFileDate=True): # Optimize the buffer for small files buffer_size = min(buffer_size, os.path.getsize(src)) if(buffer_size == 0): buffer_size = 1024 if shutil._samefile(src, dst): return for fn in [src, dst]: try: st = os.stat(fn) except OSError: pass else: if shutil.stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: shutil.copyfileobj(fsrc, fdst, buffer_size) if(perserveFileDate): shutil.copystat(src, dst)
def copyfile(src, dst, follow_symlinks=True): """Copy data from src to dst. If follow_symlinks is not set and src is a symbolic link, a new symlink will be created instead of copying the file it points to. """ if shutil._samefile(src, dst): raise shutil.SameFileError( "{!r} and {!r} are the same file".format(src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError as e: # File most likely does not exist debug(">>> {} doesn't exists [ {} ]".format(fn, e)) else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): debug(">>> creating symlink ...") os.symlink(os.readlink(src), dst) else: fs_src_type = FilesystemInfo().filesystem(src.encode('utf-8')) fs_dst_type = FilesystemInfo().filesystem( os.path.dirname(dst.encode('utf-8'))) supported_fs = ['CIFS', 'SMB2'] debug(">>> Source FS: {}".format(fs_src_type)) debug(">>> Destination FS: {}".format(fs_dst_type)) if fs_src_type in supported_fs and fs_dst_type in supported_fs: fsrc = os.open(src, os.O_RDONLY) fdst = os.open(dst, os.O_WRONLY | os.O_CREAT) CIFS_IOCTL_MAGIC = 0xCF CIFS_IOC_COPYCHUNK_FILE = IOW(CIFS_IOCTL_MAGIC, 3, c_int) # try copy file with COW support on Linux. If fail, fallback # to sendfile and if this is not available too, fallback # copyfileobj. ret = ioctl(fdst, CIFS_IOC_COPYCHUNK_FILE, fsrc) os.close(fsrc) os.close(fdst) if ret != 0: debug("!!! failed {}".format(ret)) os.close(fsrc) os.close(fdst) # Try to use sendfile if available for performance with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: if not _copyfile_sendfile(fsrc, fdst): debug("!!! failed sendfile") # sendfile is not available or failed, fallback # to copyfileobj shutil.copyfileobj(fsrc, fdst) else: with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst: if not _copyfile_sendfile(fsrc, fdst): # sendfile is not available or failed, fallback # to copyfileobj shutil.copyfileobj(src, dst) return dst
def copyfile(src, dst, follow_symlinks=True): """Copy data from src to dst. It uses windows native ``CopyFile2`` method to do so, making advantage of server-side copy where available. If this method is not available it will fallback to ``CopyFileW`` (on Windows 7 and older). Args: src (str): Source file. dst (str): Destination file. follow_symlinks (bool): If ``follow_symlinks`` is not set and ``src`` is a symbolic link, a new symlink will be created instead of copying the file it points to. Returns: str: Destination on success Raises: shutil.SpecialFileError: when source/destination is invalid. shutil.SameFileError: if ``src`` and ``dst`` are same. OSError: if file no exist IOError: if copying failed on windows API level. """ if shutil._samefile(src, dst): # Get shutil.SameFileError if available (Python 3.4+) # else fall back to original behavior using shutil.Error SameFileError = getattr(shutil, "SameFileError", shutil.Error) raise SameFileError("{!r} and {!r} are the same file".format( src, dst)) for fn in [src, dst]: try: st = os.stat(fn) except OSError: # File most likely does not exist pass else: # XXX What about other special files? (sockets, devices...) if stat.S_ISFIFO(st.st_mode): raise shutil.SpecialFileError("`%s` is a named pipe" % fn) if not follow_symlinks and os.path.islink(src): os.symlink(os.readlink(src), dst) else: kernel32 = ctypes.WinDLL('kernel32', use_last_error=True, use_errno=True) try: copyfile = kernel32.CopyFile2 except AttributeError: # on windows 7 and older copyfile = kernel32.CopyFileW copyfile.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p, ctypes.c_void_p) copyfile.restype = ctypes.HRESULT source_file = os.path.abspath(os.path.normpath(src)) dest_file = os.path.abspath(os.path.normpath(dst)) if source_file.startswith('\\\\'): source_file = 'UNC\\' + source_file[2:] if dest_file.startswith('\\\\'): dest_file = 'UNC\\' + dest_file[2:] ret = copyfile('\\\\?\\' + source_file, '\\\\?\\' + dest_file, None) if ret == 0: error = ctypes.get_last_error() if error == 0: return dst # 997 is ERROR_IO_PENDING. Why it is poping here with # CopyFileW is beyond me, but assume we can easily # ignore it as it is copying nevertheless if error == 997: return dst raise IOError("File {!r} copy failed, error: {}".format( src, ctypes.FormatError(error))) return dst