def copyFile(sourcePath, destinationPath, zstreamWrapperListener=None):
if not os.path.isfile(sourcePath):
return
if not zstreamWrapperListener:
shutil.copy2(sourcePath, destinationPath)
else:
#
# equivalent code of shutil.copy2() except uses ZStreamWrapper
#
if os.path.isdir(destinationPath):
destinationPath = os.path.join(destinationPath,
os.path.basename(sourcePath))
if shutil._samefile(sourcePath, destinationPath):
raise Error, u"`%s` and `%s` are the same file" % (
sourcePath, destinationPath) #$NON-NLS-1$
srcwrapper = None
fdst = None
try:
fsrc = open(sourcePath, u'rb') #$NON-NLS-1$
srcwrapper = ZStreamWrapper(fsrc, zstreamWrapperListener)
fdst = open(destinationPath, u'wb') #$NON-NLS-1$
shutil.copyfileobj(srcwrapper, fdst)
finally:
if fdst:
fdst.close()
if srcwrapper:
srcwrapper.close()
shutil.copystat(sourcePath, destinationPath)
def put(self, src, dst):
'''
Add src to the repo under dst. src and dst should be absolute
paths, src in the local filesystem, dst in the ETB filesystem
Return the fileref of the newly created file.
'''
if not os.path.exists(src):
error = 'File not found: %s' % src
self.log.error(error)
raise Exception(error)
gitfile = self._make_local_path(dst)
if src != gitfile:
(gdir, _) = os.path.split(gitfile)
try:
os.makedirs(gdir)
except OSError:
pass
if not (os.path.exists(gitfile)
and shutil._samefile(src, gitfile)):
shutil.copy2(src, gitfile)
return self.register(dst)
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
NOTE: this is a copy of shutil.copyfile from python 3.5, modified to be compatible
with python2.7, with the exception of the buffer size
used in copying the file contents.
"""
# noinspection PyUnresolvedReferences,PyProtectedMember
if shutil._samefile(src, dst):
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, length=COPY_BUFFSIZE)
return dst
def move(self, library, copy=False):
"""Move the item to its designated location within the library
directory (provided by destination()). Subdirectories are
created as needed. If the operation succeeds, the item's path
field is updated to reflect the new location.
If copy is True, moving the file is copied rather than moved.
Passes on appropriate exceptions if directories cannot be created
or moving/copying fails.
Note that one should almost certainly call store() and
library.save() after this method in order to keep on-disk data
consistent.
"""
dest = library.destination(self)
# Create necessary ancestry for the move.
_mkdirall(dest)
if not shutil._samefile(_syspath(self.path), _syspath(dest)):
if copy:
# copyfile rather than copy will not copy permissions
# bits, thus possibly making the copy writable even when
# the original is read-only.
shutil.copyfile(_syspath(self.path), _syspath(dest))
else:
shutil.move(_syspath(self.path), _syspath(dest))
# Either copying or moving succeeded, so update the stored path.
self.path = dest
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
NOTE: this is a copy of shutil.copyfile from python 3.5, modified to be compatible
with python2.7, with the exception of the buffer size
used in copying the file contents.
"""
# noinspection PyUnresolvedReferences,PyProtectedMember
if shutil._samefile(src, dst):
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, length=COPY_BUFFSIZE)
return dst
def copyFile(sourcePath, destinationPath, zstreamWrapperListener = None):
if not os.path.isfile(sourcePath):
return
if not zstreamWrapperListener:
shutil.copy2(sourcePath, destinationPath)
else:
#
# equivalent code of shutil.copy2() except uses ZStreamWrapper
#
if os.path.isdir(destinationPath):
destinationPath = os.path.join(destinationPath, os.path.basename(sourcePath))
if shutil._samefile(sourcePath, destinationPath):
raise Error, u"`%s` and `%s` are the same file" % (sourcePath, destinationPath) #$NON-NLS-1$
srcwrapper = None
fdst = None
try:
fsrc = open(sourcePath, u'rb') #$NON-NLS-1$
srcwrapper = ZStreamWrapper(fsrc, zstreamWrapperListener)
fdst = open(destinationPath, u'wb') #$NON-NLS-1$
shutil.copyfileobj(srcwrapper, fdst)
finally:
if fdst:
fdst.close()
if srcwrapper:
srcwrapper.close()
shutil.copystat(sourcePath, destinationPath)
def copy(self):
print("Checking paths...", end="", flush=True)
destination_path = self.network_path + self.relative_path
files_are_the_same = shutil._samefile(self.absolute_path, destination_path)
print(" DONE!")
if files_are_the_same:
print("Files are the same. Skipping copy!")
return
else:
print("Copying '" + self.absolute_path + "' to '" + destination_path + "'...", end="", flush=True)
success = False
try:
shutil.copy(self.absolute_path, destination_path)
success = True
except shutil.Error as e:
print(" FAILED!\n\tError: %s" % e)
except OSError as e:
print(" FAILED!\n\tOS Error: {m} ({n}).".format(m=e.strerror, n=e.errno))
if (not success):
print("\nBuild aborted!")
sys.exit(0)
print(" DONE!")
def copyfile(self, src, dst, *, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
size = os.stat(src).st_size
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
self.copyfileobj(fsrc, fdst, callback=self.draw_copy_progress, total=size)
return dst
def listMoving(self, targetFiles: list, listData: list) -> list:
tempList = []
toMovingList = []
for item in listData:
path = os.path.normpath(item['path'])
ext = item['extension']
if not ext:
continue
filesNames = [PurePath(x).name for x in self.scandir(path)]
for itemTarget in targetFiles:
nameTarget = PurePath(itemTarget).stem
extTarget = PurePath(itemTarget).suffix
fullName = PurePath(itemTarget).name
if extTarget.lower() in ext:
size = os.path.getsize(itemTarget)
self.totalSize += size
if fullName in filesNames or fullName in tempList:
if not shutil._samefile(itemTarget,
os.path.join(path, fullName)):
fullName = self.rename_files(
fullName, filesNames, tempList)
tempList.append(fullName)
toMovingList.append(
(itemTarget, os.path.join(path, fullName),
PurePath(itemTarget).name, size))
return sorted(toMovingList, key=lambda x: x[3])
def shutil_copyfile(src, dst):
"""Copy data from src to dst"""
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
elif not os.path.exists(src) or os.path.isdir(src):
return
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
try:
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
except NameError:
raise shutil.Error("`%s` is a named pipe" % fn)
BUFFER_SIZE = 128 * 1024
try:
with open(src, "rb") as fin, open(dst, "wb") as fout:
for x in iter(lambda: fin.read(BUFFER_SIZE), ""):
fout.write(x)
except Exception as e:
raise
def copyfile(src, dst, *, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
if _sys == 'Linux':
subprocess.check_call(['cp', str(src), str(dst)])
else:
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, length=16 * 1024 * 1024)
return dst
def shutil_copyfile(src, dst):
"""Copy data from src to dst"""
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
elif not os.path.exists(src) or os.path.isdir(src):
return
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
try:
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
except NameError:
raise shutil.Error("`%s` is a named pipe" % fn)
BUFFER_SIZE = 128*1024
try:
with open(src, "rb") as fin, open(dst, "wb") as fout:
for x in iter(lambda: fin.read(BUFFER_SIZE), ""):
fout.write(x)
except Exception as e:
raise
def copy_file(src, dst, buffer_size=10485760, preserve_file_date=True):
"""
Copies a file to a new location.
Much faster performance than Apache Commons due to use of larger buffer.
:param src: Source file path
:param dst: Destination file path
:param buffer_size: Buffer size to use during copy
:param preserve_file_date: Preserve the original file date
"""
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if buffer_size == 0:
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`{0}` and `{1}` are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError: # File most likely does not exist
pass
else: # XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError(
"`{}` is a named pipe".format(fn))
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
if preserve_file_date:
shutil.copystat(src, dst)
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
It uses windows `xcopy` method to do so, making advantage of
server-side copy where available. Different other methods can be used
as robocopy is probably faster, but robocopy doesn't support renaming
destination file when copying just one file :( Shame on you Microsoft.
"""
from subprocess import call
try:
# Python 3.3+
from subprocess import DEVNULL
except ImportError:
# Backwards compatibility
DEVNULL = open(os.devnull, 'w')
# from pathlib import Path, PureWindowsPath
if shutil._samefile(src, dst):
# Get shutil.SameFileError if available (Python 3.4+)
# else fall back to original behavior using shutil.Error
SameFileError = getattr(shutil, "SameFileError", shutil.Error)
raise SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
# call(["xcopy", src, dst], stdin=DEVNULL, stdout=DEVNULL)
cmd = [
"echo", "f", "|", "xcopy", "/y", "/h", "/r",
src.replace('/', '\\'),
dst.replace('/', '\\')
]
# cmd = ["copy", "/B", "/Y",
# src.replace('/', '\\'), dst.replace('/', '\\')]
call(cmd, stdin=DEVNULL, stdout=DEVNULL, shell=True)
# call(["robocopy",
# os.path.dirname(src),
# os.path.dirname(dst),
# os.path.basename(src),
# "/njh", "/njs", "/ndl", "/nc", "/ns", "/nfl"])
# os.rename(os.path.join(
# os.path.dirname(dst), os.path.basename(src)),
# dst)
# call(["copy", src, dst, "/B", "/Y"])
return dst
def cp( self , dst , basename = None , overwrite = False ): src = self dst = __mod__.Shell( dst , delegated = True ) if basename == None: dst = dst( self.BASENAME ) else: dst = dst( basename ) if shutil._samefile(src, dst): raise shutil.Error( "`%s` and `%s` are the same file" % (src, tmp_dst) ) #print 3334444 , 'shell.cp1' , dst #print 3334444 , 'shell.cp2' , dst.PARENT dst.PARENT.MKDIR #Optimization buffer_size = min( 10485760 , os.path.getsize(src) ) or 1024 #No copia si son iguales if not overwrite and dst.ISFILE: if self.cmp( dst ): print 'Same file, no copy %s' % dst return dst with open(src, 'rb') as fsrc: with open(dst, 'wb') as fdst: shutil.copyfileobj( fsrc, fdst, buffer_size ) shutil.copystat( src, dst ) return dst
def copyFile(FileSource, Destination, BufferSize=1024 * 1024 * 10, PerserveFileDate=True):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param FileSource: Source File
@param Destination: Destination File (not file path)
@param BufferSize: Buffer size to use during copy
@param PerserveFileDate: Preserve the original file date
'''
# Optimize the buffer for small files
BufferSize = min(BufferSize, os.path.getsize(FileSource))
if(BufferSize == 0):
BufferSize = 1024
if shutil._samefile(FileSource, Destination):
LogBrowser("`{0}` and `{1}` are the same file".format(FileSource, Destination), "ERROR")
raise shutil.Error("`{0}` and `{1}` are the same file".format(FileSource, Destination))
for FileName in [FileSource, Destination]:
try:
st = os.stat(FileName)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
LogBrowser("`{0}` is a named pipe".format(FileName), "ERROR")
raise shutil.SpecialFileError("`{0}` is a named pipe".format(FileName))
with open(FileSource, 'rb') as fFileSource:
with open(Destination, 'wb') as fDestination:
shutil.copyfileobj(fFileSource, fDestination, BufferSize)
if(PerserveFileDate):
shutil.copystat(FileSource, Destination)
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
# Try to use sendfile if available for performance
if not _copyfile_sendfile(fsrc, fdst):
# sendfile is not available or failed, fallback to copyfileobj
shutil.copyfileobj(fsrc, fdst)
return dst
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
if shutil._samefile(src, dst):
raise shutil.SameFileError(
"{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
# Try to use sendfile if available for performance
if not _copyfile_sendfile(fsrc, fdst):
# sendfile is not available or failed, fallback to copyfileobj
shutil.copyfileobj(fsrc, fdst)
return dst
def move(src,
dst,
overwrite=False,
make_safe_path=get_safe_path,
name_pairs=[]):
"""Recursively move a file or directory to another location. This is
similar to the Unix "mv" command.
If the destination is a directory or a symlink to a directory, the source
is moved inside the directory. The destination path must not already
exist.
If the destination already exists but is not a directory, it may be
overwritten depending on os.rename() semantics.
If the destination is on our current filesystem, then rename() is used.
Otherwise, src is copied to the destination and then removed.
A lot more could be done here... A look at a mv.c shows a lot of
the issues this implementation glosses over.
"""
real_dst = dst
if os.path.isdir(dst):
if _samefile(src, dst):
# We might be on a case insensitive filesystem,
# perform the rename anyway.
os.rename(src, dst)
name_pairs.append((src, dst))
return
real_dst = os.path.join(dst, _basename(src))
if not overwrite:
real_dst = make_safe_path(real_dst)
try:
os.rename(src, real_dst)
name_pairs.append((src, real_dst))
except OSError:
if os.path.isdir(src):
if _destinsrc(src, dst):
raise Error("Cannot move a directory '%s' into itself '%s'." %
(src, dst))
for done in copytree(src,
real_dst,
symlinks=True,
overwrite=overwrite,
make_safe_path=make_safe_path):
yield done
rmtree(src)
name_pairs.append((src, real_dst))
else:
for done in copy2(src,
real_dst,
symlinks=True,
overwrite=overwrite,
make_safe_path=make_safe_path):
yield done
os.unlink(src)
name_pairs.append((src, real_dst))
def copyfile(src, dst, follow_symlinks=True):
"""Copy data from src to dst.
It uses windows native CopyFileW method to do so, making advantage of
server-side copy where available.
"""
if shutil._samefile(src, dst):
# Get shutil.SameFileError if available (Python 3.4+)
# else fall back to original behavior using shutil.Error
SameFileError = getattr(shutil, "SameFileError", shutil.Error)
raise SameFileError(
"{!r} and {!r} are the same file".format(src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
kernel32 = ctypes.WinDLL('kernel32',
use_last_error=True, use_errno=True)
copyfile = kernel32.CopyFile2
copyfile.argtypes = (ctypes.c_wchar_p,
ctypes.c_wchar_p,
ctypes.c_void_p)
copyfile.restype = ctypes.HRESULT
source_file = os.path.normpath(src)
dest_file = os.path.normpath(dst)
if source_file.startswith('\\\\'):
source_file = 'UNC\\' + source_file[2:]
if dest_file.startswith('\\\\'):
dest_file = 'UNC\\' + dest_file[2:]
ret = copyfile('\\\\?\\' + source_file,
'\\\\?\\' + dest_file, None)
if ret != 0:
error = ctypes.get_last_error()
if error == 0:
return dst
# 997 is ERROR_IO_PENDING. Why it is poping here with
# CopyFileW is beyond me, but assume we can easily
# ignore it as it is copying nevertheless
if error == 997:
return dst
raise IOError(
"File {!r} copy failed, error: {}".format(src, error))
return dst
def copy_file(src, dst, buffer_size=10485760, perserveFileDate=None):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param src: Source File
@param dst: Destination File (not file path)
@param buffer_size: Buffer size to use during copy
@param perserveFileDate: Preserve the original file date
'''
# Check to make sure destination directory exists. If it doesn't create the directory
dstParent, dstFileName = os.path.split(dst)
if (not (os.path.exists(dstParent))):
os.makedirs(dstParent)
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if (buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
src_fh = os.open(src, os.O_RDONLY | os.O_SYNC)
dst_fh = os.open(dst,
os.O_CREAT | os.O_SYNC | os.O_TRUNC | os.O_WRONLY)
if src_fh != None and dst_fh != None:
while True:
buffer = os.read(src_fh, lock.BUFFER_SIZE)
if buffer == '':
break
os.write(dst_fh, buffer)
if src_fh:
os.close(src_fh)
if dst_fh:
os.close(dst_fh)
# shutil.copyfileobj(fsrc, fdst, buffer_size)
#if lock.debug: lock.info("Lock","\n".join(f.readlines()))
if (perserveFileDate):
shutil.copystat(src, dst)
def copy_file(src, dst, mk_dst=True):
# mk_dst: make directory if doesn't exists
if shutil._samefile(src, dst):
msg = "{!r} and {!r} are the same file".format(src, dst)
raise shutil.SameFileError(msg)
else:
if mk_dst:
paths.mkdir(os.path.dirname(dst))
source_size = os.stat(src).st_size
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst, source_size)
return dst
def copy_pro(src, dst, *, follow_symlinks=True,
force=False, progress_min_size=1e7):
"""
Copy data from src to dst.
show progress bar in terminal for large files (> 10 MB by default)
If follow_symlinks is not set and src is a symbolic link, a new
symlink will be created instead of copying the file it points to.
"""
src = Path(src)
dst = Path(dst)
# file/dir -> dir
if dst.is_dir():
dst = dst / src.name
# same file error?
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file"
.format(src, dst))
# destination exists error?
if dst.exists():
if force:
dst.unlink()
else:
raise IOError(f"copy failed, destination already exists: {dst}, use force=True to overwrite")
size = os.stat(src).st_size
# symlink copy?
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
# file copy
callback = print_copy_progress if size > progress_min_size else None
with src.open(mode='rb') as fsrc:
with dst.open(mode='wb+') as fdst: # open or create
__copyfileobj(fsrc, fdst, callback=callback, total=size)
# copy permissions
shutil.copymode(src, dst)
# check success by size
if not cmp(src, dst, shallow=True):
raise OSError(f'Error, copy failed {src}: {size} b \
-> {dst}: {os.stat(dst).st_size} b')
return dst
def CopyFile2(src, dst, buffer_size=10485760, perserveFileDate=True):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param src: Source File
@param dst: Destination File (not file path)
@param buffer_size: Buffer size to use during copy
@param perserveFileDate: Preserve the original file date
'''
try:
# Check to make sure destination directory exists. If it doesn't create the directory
dstParent, dstFileName = os.path.split(dst)
if(not(os.path.exists(dstParent))):
os.makedirs(dstParent)
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if(buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
if(perserveFileDate):
CopyStat(src, dst)
return True
except Exception, e:
print "Error:", e
if os.name == 'nt':
try:
WinCopy(src, dst)
return True
except Exception, e1:
print "WinCopy Error:", e1
def shutil_move(src, dst):
"""Recursively move a file or directory to another location. This is
similar to the Unix "mv" command. Return the file or directory's
destination.
If the destination is a directory or a symlink to a directory, the source
is moved inside the directory. The destination path must not already
exist.
If the destination already exists but is not a directory, it may be
overwritten depending on os.rename() semantics.
If the destination is on our current filesystem, then rename() is used.
Otherwise, src is copied to the destination and then removed. Symlinks are
recreated under the new name if os.rename() fails because of cross
filesystem renames.
A lot more could be done here... A look at a mv.c shows a lot of
the issues this implementation glosses over.
"""
real_dst = dst
if _os.path.isdir(dst):
if shutil._samefile(src, dst):
# We might be on a case insensitive filesystem,
# perform the rename anyway.
_os.rename(src, dst)
return
real_dst = _os.path.join(dst, shutil._basename(src))
if _os.path.exists(real_dst):
raise shutil.Error("Destination path '%s' already exists" % real_dst)
try:
_os.rename(src, real_dst)
except OSError:
if _os.path.islink(src):
linkto = _os.readlink(src)
_os.symlink(linkto, real_dst)
_os.unlink(src)
elif _os.path.isdir(src):
if shutil._destinsrc(src, dst):
raise shutil.Error("Cannot move a directory '%s' into itself '%s'." % (src, dst))
shutil.copytree(src, real_dst, symlinks=True)
shutil.rmtree(src)
else:
shutil.copy2(src, real_dst)
_os.unlink(src)
return real_dst
def save(self):
if self.state_dir:
state_filename = os.path.join(self.state_dir, "torrents.state")
resume_filename = os.path.join(self.state_dir,
"torrents.fastresume")
self.__write(self.torrents.keys(), state_filename, resume_filename)
else:
for torrent_id in self.torrents:
state_filename = torrent_id + ".state"
resume_filename = torrent_id + ".fastresume"
self.__write([torrent_id], state_filename, resume_filename)
for torrent_id in self.torrents:
new_filename = os.path.join(self.state_dir,
torrent_id + ".torrent")
if not _samefile(self.torrents[torrent_id]["filename"],
new_filename):
copy(self.torrents[torrent_id]["filename"], new_filename)
def copyFile(src, dst, buffer_size=10485760, perserveFileDate=True):
'''
Copies a file to a new location. Much faster performance than Apache Commons due to use of larger buffer
@param src: Source File
@param dst: Destination File (not file path)
@param buffer_size: Buffer size to use during copy
@param perserveFileDate: Preserve the original file date
'''
# Check to make sure destination directory exists. If it doesn't create the directory
dstParent, dstFileName = os.path.split(dst)
if(not(os.path.exists(dstParent))):
os.makedirs.(dstParent)
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if(buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
# with open(src, 'rb') as fsrc:
# with open(dst, 'wb') as fdst:
# shutil.copyfileobj(fsrc, fdst, buffer_size)
fsrc = open(src, 'rb')
try:
fdst = open(dst, 'wb')
try:
shutil.copyfileobj(fsrc, fdst, buffer_size)
finally:
fdst.close()
finally:
fsrc.close()
if(perserveFileDate):
shutil.copystat(src, dst)
def copyfile(src, dst):
"""Copy data from src to dst"""
if _samefile(src, dst):
raise Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst)
def copyfile(src, dst):
"""Copy data from src to dst"""
if _samefile(src, dst):
raise Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst)
def copyfile_custom(src, dst):
"""Copy data from src to dst"""
if _samefile(src, dst):
raise Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
try:
raise SpecialFileError("`%s` is a named pipe" % fn)
except NameError:
raise Error("`%s` is a named pipe" % fn)
try:
# Windows
O_BINARY = os.O_BINARY
except:
O_BINARY = 0
READ_FLAGS = os.O_RDONLY | O_BINARY
WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY
BUFFER_SIZE = 128*1024
try:
fin = os.open(src, READ_FLAGS)
fout = os.open(dst, WRITE_FLAGS)
for x in iter(lambda: os.read(fin, BUFFER_SIZE), ""):
os.write(fout, x)
except Exception as e:
raise e
finally:
try:
os.close(fin)
os.close(fout)
except:
pass
def copyfile_custom(src, dst):
"""Copy data from src to dst"""
if _samefile(src, dst):
raise Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
try:
raise SpecialFileError("`%s` is a named pipe" % fn)
except NameError:
raise Error("`%s` is a named pipe" % fn)
try:
# Windows
O_BINARY = os.O_BINARY
except:
O_BINARY = 0
READ_FLAGS = os.O_RDONLY | O_BINARY
WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | O_BINARY
BUFFER_SIZE = 128 * 1024
try:
fin = os.open(src, READ_FLAGS)
fout = os.open(dst, WRITE_FLAGS)
for x in iter(lambda: os.read(fin, BUFFER_SIZE), ""):
os.write(fout, x)
except Exception as e:
raise
finally:
try:
os.close(fin)
os.close(fout)
except:
pass
def copyfile(src, dst):
"""Copy data from src to dst"""
if _samefile(src, dst):
raise Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]: # pylint: disable=invalid-name
try:
st = os.stat(fn) # pylint: disable=invalid-name
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("`%s` is a named pipe" % fn)
with open(src, "rb") as fsrc:
with open(dst, "wb") as fdst:
for done in copyfileobj(fsrc, fdst):
yield done
def move(src, dst):
"""Recursively move a file or directory to another location. This is
similar to the Unix "mv" command.
If the destination is a directory or a symlink to a directory, the source
is moved inside the directory. The destination path must not already
exist.
If the destination already exists but is not a directory, it may be
overwritten depending on os.rename() semantics.
If the destination is on our current filesystem, then rename() is used.
Otherwise, src is copied to the destination and then removed.
A lot more could be done here... A look at a mv.c shows a lot of
the issues this implementation glosses over.
"""
real_dst = dst
if os.path.isdir(dst):
if _samefile(src, dst):
# We might be on a case insensitive filesystem,
# perform the rename anyway.
os.rename(src, dst)
return
real_dst = os.path.join(dst, _basename(src))
if os.path.exists(real_dst):
raise Error("Destination path '%s' already exists" % real_dst)
try:
os.rename(src, real_dst)
except OSError:
if os.path.isdir(src):
if _destinsrc(src, dst):
msg = "Cannot move a directory '%s' into itself '%s'." % (src,
dst)
raise Error(msg)
copytree(src, real_dst, symlinks=True)
rmtree(src)
else:
copy(src, real_dst)
os.unlink(src)
def copy_file(self, src, dst):
""" Copy given source to destination
Arguments:
src (str): the source file which needs to be copied
dst (str): the destination of the sourc file
Returns:
None
"""
src = os.path.normpath(src)
dst = os.path.normpath(dst)
self.log.debug("Copying file ... {} -> {}".format(src, dst))
dirname = os.path.dirname(dst)
try:
os.makedirs(dirname)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
self.log.critical("An unexpected error occurred.")
six.reraise(*sys.exc_info())
# copy file with speedcopy and check if size of files are simetrical
while True:
if not shutil._samefile(src, dst):
copyfile(src, dst)
else:
self.log.critical("files are the same {} to {}".format(
src, dst))
os.remove(dst)
try:
shutil.copyfile(src, dst)
self.log.debug("Copying files with shutil...")
except OSError as e:
self.log.critical("Cannot copy {} to {}".format(src, dst))
self.log.critical(e)
six.reraise(*sys.exc_info())
if str(getsize(src)) in str(getsize(dst)):
break
def copy_file(src, dest, buffer_size=DEFAULT_BUFFER):
'''Copy operations can be optimized by setting the buffer size for read
operations and that's just what copy_file does. copy_file has a default
buffer size of 256 KB, which increases the speed of file copying in most
cases. For smaller files the buffer size is reduced to the file size or a
minimum of MINIMUM_BUFFER (1 KB).
See also:
http://blogs.blumetech.com/blumetechs-tech-blog/2011/05/faster-python-file-copy.html
https://stackoverflow.com/questions/22078621/python-how-to-copy-files-fast/28129677#28129677
Arguments:
src (str): source file to copy
dest (str): destination file path
buffer_size (int): Number of bytes to buffer
'''
if shutil._samefile(src, dest):
raise OSError('Source and destination can not be the same...')
destdir = os.path.dirname(dest)
if not os.path.exists(destdir):
os.makedirs(destdir)
buffer_size = optimize_buffer(src, buffer_size)
try:
i_file = os.open(src, RFLAGS)
i_stat = os.fstat(i_file)
o_file = os.open(dest, WFLAGS, i_stat.st_mode)
while True:
b = os.read(i_file, buffer_size)
if b == '':
break
os.write(o_file, b)
finally:
suppress(os.close, i_file)
suppress(os.close, o_file)
def copyFile(src, dst, buffer_size=10485760, perserveFileDate=True):
# Optimize the buffer for small files
buffer_size = min(buffer_size, os.path.getsize(src))
if(buffer_size == 0):
buffer_size = 1024
if shutil._samefile(src, dst):
return
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
pass
else:
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
if(perserveFileDate):
shutil.copystat(src, dst)
def copyfile(src, dst, *, follow_symlinks=True):
if shutil._samefile(src, dst):
raise shutil.SameFileError("{!r} and {!r} are the same file".format(
src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
pass
else:
if shutil.stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
if not follow_symlinks and os.path.islink(src):
os.symlink(os.readlink(src), dst)
else:
size = os.stat(src).st_size
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
copyfileobj(fsrc, fdst, callback=copy_progress, total=size)
return dst
def fast_copyfile(src, dst, buffer_size=1024 * 1024):
"""
Copy data from src to dst - reimplemented with a buffer size
Note that this function is simply a copy of the function from the official python shutils.py file, but
with an increased (configurable) buffer size fed into copyfileobj instead of the original 16kb one
"""
if shutil._samefile(src, dst):
raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
for fn in [src, dst]:
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
# XXX What about other special files? (sockets, devices...)
if stat.S_ISFIFO(st.st_mode):
raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
with open(src, 'rb') as fsrc:
with open(dst, 'wb') as fdst:
shutil.copyfileobj(fsrc, fdst, buffer_size)
def move(self, library, copy=False, in_album=False):
"""Move the item to its designated location within the library
directory (provided by destination()). Subdirectories are
created as needed. If the operation succeeds, the item's path
field is updated to reflect the new location.
If copy is True, moving the file is copied rather than moved.
If in_album is True, then the track is treated as part of an
album even if it does not yet have an album_id associated with
it. (This allows items to be moved before they are added to the
database, a performance optimization.)
Passes on appropriate exceptions if directories cannot be created
or moving/copying fails.
Note that one should almost certainly call store() and
library.save() after this method in order to keep on-disk data
consistent.
"""
dest = library.destination(self, in_album=in_album)
# Create necessary ancestry for the move.
util.mkdirall(dest)
if not shutil._samefile(syspath(self.path), syspath(dest)):
if copy:
# copyfile rather than copy will not copy permissions
# bits, thus possibly making the copy writable even when
# the original is read-only.
shutil.copyfile(syspath(self.path), syspath(dest))
else:
shutil.move(syspath(self.path), syspath(dest))
# Either copying or moving succeeded, so update the stored path.
self.path = dest
def copyfile_custom(src, dst):
"""Copy data from src to dst."""
def special_file(fn):
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("{!r} is a named pipe".format(fn))
if _samefile(src, dst):
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
special_file(src)
special_file(dst)
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
while True:
buf = fsrc.read(BUFFER_SIZE)
if not buf:
break
fdst.write(buf)
def copyfile_custom(src, dst):
"""Copy data from src to dst."""
def special_file(fn):
try:
st = os.stat(fn)
except OSError:
# File most likely does not exist
pass
else:
if stat.S_ISFIFO(st.st_mode):
raise SpecialFileError("{!r} is a named pipe".format(fn))
if _samefile(src, dst):
raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
special_file(src)
special_file(dst)
with open(src, 'rb') as fsrc, open(dst, 'wb') as fdst:
while True:
buf = fsrc.read(BUFFER_SIZE)
if not buf:
break
fdst.write(buf)
def samefile(p1, p2):
"""Safer equality for paths."""
return shutil._samefile(syspath(p1), syspath(p2))
def samefile(p1, p2):
"""Safer equality for paths."""
return shutil._samefile(syspath(p1), syspath(p2))
def isSame(src, dst):
return shutil._samefile(src, dst)
def samefile(p1, p2):
"""Safer equality for paths."""
if p1 == p2:
return True
return shutil._samefile(syspath(p1), syspath(p2))