Example #1
0
def shutil_copyfile(src, dst):
    """Copy data from src to dst"""
    if shutil._samefile(src, dst):
        raise shutil.Error("`%s` and `%s` are the same file" % (src, dst))
    elif not os.path.exists(src) or os.path.isdir(src):
        return

    for fn in [src, dst]:
        try:
            st = os.stat(fn)
        except OSError:
            # File most likely does not exist
            pass
        else:
            # XXX What about other special files? (sockets, devices...)
            if stat.S_ISFIFO(st.st_mode):
                try:
                    raise shutil.SpecialFileError("`%s` is a named pipe" % fn)
                except NameError:
                    raise shutil.Error("`%s` is a named pipe" % fn)

    BUFFER_SIZE = 128 * 1024
    try:
        with open(src, "rb") as fin, open(dst, "wb") as fout:
            for x in iter(lambda: fin.read(BUFFER_SIZE), ""):
                fout.write(x)
    except Exception as e:
        raise
Example #2
0
    def move(self, src, dst, copy_function=None):
        """Rename a file or directory.

        Args:
          src: (str) source file or directory
          dst: (str) if the src is a directory, the dst must not already exist
          copy_function: replacement for copy2 if copying is needed.
            New in Python 3.5. New in pyfakefs 2.9.
        """

        def _destinsrc(src, dst):
            src = os.path.abspath(src)
            dst = os.path.abspath(dst)
            if not src.endswith(self.filesystem.path_separator):
                src += os.path.sep
            if not dst.endswith(self.filesystem.path_separator):
                dst += self.filesystem.path_separator
            return dst.startswith(src)

        if copy_function is not None:
            if sys.version_info < (3, 5):
                raise TypeError("move() got an unexpected keyword argument 'copy_function")
        else:
            copy_function = self.copy2

        src = self.filesystem.NormalizePath(src)
        dst = self.filesystem.NormalizePath(dst)
        if src == dst:
            return dst

        source_is_dir = stat.S_ISDIR(self.filesystem.GetObject(src).st_mode)
        if source_is_dir:
            dst = self.filesystem.JoinPaths(dst, os.path.basename(src))
            if self.filesystem.Exists(dst):
                raise shutil.Error("Destination path '%s' already exists" % dst)

        try:
            self.filesystem.RenameObject(src, dst)
        except OSError:
            if source_is_dir:
                if _destinsrc(src, dst):
                    raise shutil.Error("Cannot move a directory '%s' into itself"
                                       " '%s'." % (src, dst))
                self._copytree(src, dst, copy_function=copy_function, symlinks=True)
                self.rmtree(src)
            else:
                copy_function(src, dst)
                self.filesystem.RemoveObject(src)
        return dst
Example #3
0
def shutil_move(src, dst):
    """Recursively move a file or directory to another location. This is
    similar to the Unix "mv" command. Return the file or directory's
    destination.

    If the destination is a directory or a symlink to a directory, the source
    is moved inside the directory. The destination path must not already
    exist.

    If the destination already exists but is not a directory, it may be
    overwritten depending on os.rename() semantics.

    If the destination is on our current filesystem, then rename() is used.
    Otherwise, src is copied to the destination and then removed. Symlinks are
    recreated under the new name if os.rename() fails because of cross
    filesystem renames.

    A lot more could be done here...  A look at a mv.c shows a lot of
    the issues this implementation glosses over.

    """
    real_dst = dst
    if _os.path.isdir(dst):
        if shutil._samefile(src, dst):
            # We might be on a case insensitive filesystem,
            # perform the rename anyway.
            _os.rename(src, dst)
            return

        real_dst = _os.path.join(dst, shutil._basename(src))
        if _os.path.exists(real_dst):
            raise shutil.Error("Destination path '%s' already exists" % real_dst)
    try:
        _os.rename(src, real_dst)
    except OSError:
        if _os.path.islink(src):
            linkto = _os.readlink(src)
            _os.symlink(linkto, real_dst)
            _os.unlink(src)
        elif _os.path.isdir(src):
            if shutil._destinsrc(src, dst):
                raise shutil.Error("Cannot move a directory '%s' into itself '%s'." % (src, dst))
            shutil.copytree(src, real_dst, symlinks=True)
            shutil.rmtree(src)
        else:
            shutil.copy2(src, real_dst)
            _os.unlink(src)
    return real_dst
Example #4
0
def merge_tree(src, dst):
    """
  Similar to shutil.copytree, but does not complain when the destination
  directory already exists.
  """
    names = os.listdir(src)
    if not os.path.exists(dst):
        os.makedirs(dst)
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                merge_tree(srcname, dstname)
            else:
                shutil.copy2(srcname, dstname)
        except (IOError, os.error) as why:
            errors.append((srcname, dstname, str(why)))
        except shutil.Error as err:
            errors.extend(err.args[0])
    try:
        shutil.copystat(src, dst)
    except OSError as why:
        if WindowsError is not None and isinstance(why, WindowsError):
            # Copying file access times may fail on Windows
            pass
        else:
            errors.extend((src, dst, str(why)))
    if errors:
        raise shutil.Error(errors)
Example #5
0
 def mergeDir(src, dst, symlinks=False):
     names = os.listdir(src)
     if not os.path.isdir(dst):
         os.makedirs(dst)
     errors = []
     for name in names:
         srcname = os.path.join(src, name)
         dstname = os.path.join(dst, name)
         try:
             if symlinks and os.path.islink(srcname):
                 linkto = os.readlink(srcname)
                 os.symlink(linkto, dstname)
             elif os.path.isdir(srcname):
                 JUtils.mergeDir(srcname, dstname, symlinks)
             else:
                 if os.path.isfile(dstname):
                     os.remove(dstname)
                 shutil.copy2(srcname, dstname)
                 # XXX What about devices, sockets etc.?
         except (IOError, os.error) as why:
             errors.append((srcname, dstname, str(why)))
         # catch the Error from the recursive copytree so that we can
         # continue with other files
         except OSError as err:
             errors.extend(err.args[0])
     if errors:
         raise shutil.Error(errors)
Example #6
0
def smart_copytree(src, dst, symlinks=False, ignore=None):
    names = os.listdir(src)
    if ignore is not None:
        ignored_names = ignore(src, names)
    else:
        ignored_names = set()
    print(src, dst)
    if not os.path.isdir(dst):
        os.makedirs(dst)
    errors = []
    for name in names:
        if name in ignored_names:
            continue
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                smart_copytree(srcname, dstname, symlinks, ignore)
            else:
                # Will raise a SpecialFileError for unsupported file types
                shutil.copy(srcname, dstname)
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])
        except EnvironmentError as why:
            errors.append((srcname, dstname, str(why)))
    if errors:
        raise shutil.Error(errors)
Example #7
0
def copytree_override(src, dst, symlinks=False, ignore=None):
    names = os.listdir(src)
    if ignore is not None:
        ignored_names = ignore(src, names)
    else:
        ignored_names = set()
    if not os.path.isdir(dst):
        os.makedirs(dst)
    errors = []
    for name in names:
        if name in ignored_names:
            continue
        src_name = os.path.join(src, name)
        dst_name = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(src_name):
                link_to = os.readlink(src_name)
                os.symlink(link_to, dst_name)
            elif os.path.isdir(src_name):
                copytree_override(src_name, dst_name, symlinks, ignore)
            else:
                shutil.copy2(src_name, dst_name)
        except shutil.Error as err:
            errors.extend(err.args[0])
        except EnvironmentError as why:
            errors.append((src_name, dst_name, str(why)))
    try:
        shutil.copystat(src, dst)
    except OSError as why:
        if WindowsError is not None and isinstance(why, WindowsError):
            pass
        else:
            errors.append((src, dst, str(why)))
    if errors:
        raise shutil.Error(errors)
Example #8
0
 def merge_dir(src, dst, symlinks=False, ignorHidden=True):
     if not os.path.exists(src):
         print('src dir not exists on mergeing')
         return
     names = os.listdir(src)
     if not os.path.isdir(dst):
         os.makedirs(dst)
     errors = []
     for name in names:
         if ignorHidden and name[0] == '.':
             continue
         srcname = os.path.join(src, name)
         dstname = os.path.join(dst, name)
         try:
             if symlinks and os.path.islink(srcname):
                 os.symlink(os.readlink(srcname), dstname)
             elif os.path.isdir(srcname):
                 JFileUtils.merge_dir(srcname, dstname, symlinks)
             else:
                 if os.path.isfile(dstname):
                     os.remove(dstname)
                 shutil.copy2(srcname, dstname)
         except (IOError, os.error) as why:
             errors.append((srcname, dstname, str(why)))
         except OSError as err:
             errors.extend(err.args[0])
     if errors:
         raise shutil.Error(errors)
Example #9
0
    def testShareUpload(self, map_drive, unmap_drive, copy, get_file_name):
        class TestCredProvider(log_copy.LogCopyCredentials):
            def GetUsername(self):
                return 'test_user'

            def GetPassword(self):
                return 'test_pass'

        log_copy.LogCopyCredentials = TestCredProvider

        log_host = 'log-host.example.com'
        get_file_name.return_value = 'log.txt'
        self.lc.ShareCopy(self.log_file, log_host)
        map_drive.assert_called_with(mock.ANY, 'l:', 'log-host.example.com',
                                     'test_user', 'test_pass')
        copy.assert_called_with(self.log_file, 'log.txt')
        unmap_drive.assert_called_with(mock.ANY, 'l:')
        # map error
        map_drive.return_value = None
        self.assertRaises(log_copy.LogCopyError, self.lc.ShareCopy,
                          self.log_file, log_host)
        # copy error
        map_drive.return_value = True
        copy.side_effect = shutil.Error()
        self.assertRaises(log_copy.LogCopyError, self.lc.ShareCopy,
                          self.log_file, log_host)
Example #10
0
    def copyConfig(self, src, dst):
        """Recursively copy the examples directories
    """
        if not os.path.isdir(dst):
            raise shutil.Error('Destination is not a directory')

        self.copies.extend(self.copyfile('gmakefile.test', dst))
Example #11
0
def copytree(src, dst, copy_function=shutil.copy2, symlinks=False):
    "Implementation adapted from https://docs.python.org/3/library/shutil.html#copytree-example'."
    os.makedirs(dst)
    names = os.listdir(src)
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                copytree(srcname, dstname, copy_function, symlinks)
            else:
                copy_function(srcname, dstname)
        except OSError as why:
            raise
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            raise
            errors.extend(err.args[0])
    if errors:
        raise shutil.Error(errors)
Example #12
0
def copyProjects(projectPath):
    dstPath = getWorkspaceLocation()
    print "copying project to", dstPath
    names = os.listdir(projectPath)

    errors = []
    for name in names:
        if name != ".metadata" and name != ".closet":
            srcname = os.path.join(projectPath, name)
            dstname = os.path.join(dstPath, name)
            try:
                if os.path.isdir(srcname):
                    copyProjectFolder(srcname, dstname)
                else:
                    shutil.copy(srcname, dstname)
            except (IOError, os.error) as why:
                errors.append((srcname, dstname, str(why)))
            # catch the Error from the recursive copytree so that we can
            # continue with other files
            except shutil.Error as err:
                errors.extend(err.args[0])
    if errors:
        raise shutil.Error(errors)

    return dstPath
Example #13
0
 def copytree_rec(src, dst):
     names = os.listdir(src)
     makedirs(dst, overwrite=overwrite)
     errors = []
     for name in names:
         srcname = os.path.join(src, name)
         if srcname in skip:
             continue
         dstname = os.path.join(dst, name)
         try:
             if symlinks and os.path.islink(srcname):
                 remove_if_overwriting(dstname)
                 linkto = os.readlink(srcname)
                 os.symlink(linkto, dstname)
             elif os.path.isdir(srcname):
                 copytree_rec(srcname, dstname)
             else:
                 remove_if_overwriting(dstname)
                 shutil.copy2(srcname, dstname)
             # XXX What about devices, sockets etc.?
         except (IOError, OSError) as why:
             errors.append((srcname, dstname, str(why)))
         # catch the Error from the recursive copytree so that we can
         # continue with other files
         except shutil.Error as err:
             errors.extend(err.args[0])
     try:
         shutil.copystat(src, dst)
     except WindowsError:
         pass  # Ignore errors due to limited Windows copystat support
     except OSError as why:
         errors.append((src, dst, str(why)))
     if errors:
         raise shutil.Error(errors)
Example #14
0
def copytree(src, dst):
    """Copy directory tree recursively.
    
    Alternative implementation of shutil.copytree which allows to copy into
    existing directories. Directories which does not exist are created and
    existing directories are populated with files and folders.
    If destination directory path does not exist, it will attempt to create
    the entire directory tree up to that level.
    
    :param src: source directory path
    :param dst: destination directory path
    :return: destination directory path
    """
    os.makedirs(dst, exist_ok=True)
    errors = []
    for name in os.listdir(src):
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if os.path.isdir(srcname):
                copytree(srcname, dstname)
            else:
                shutil.copy(srcname, dstname)
        except shutil.Error as err:
            errors.extend(err.args[0])
        except OSError as err:
            errors.append((srcname, dstname, str(err)))
    if errors:
        raise shutil.Error(errors)
    return dst
Example #15
0
def copyProjectFolder(projectPath, baseName):
    # get project contents
    tempPath = getWorkspaceLocation()
    projPath = os.path.join(tempPath, baseName)
    if os.path.isdir(projPath):
        shutil.rmtree(projPath, ignore_errors=True)
    print "copying project to", projPath
    names = os.listdir(projectPath)

    os.makedirs(projPath)
    errors = []
    for name in names:
        srcname = os.path.join(projectPath, name)
        dstname = os.path.join(projPath, name)
        try:
            if os.path.isdir(srcname):
                copyProjectFolder(srcname, dstname)
            else:
                shutil.copy(srcname, dstname)
        except (IOError, os.error) as why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])
    if errors:
        raise shutil.Error(errors)

    return projPath
Example #16
0
    def copyExamples(self, src, dst):
        """Recursively copy the examples directories
    """
        if not os.path.isdir(dst):
            raise shutil.Error('Destination is not a directory')

        self.copyfile(os.path.join(src, 'makefile'), dst)
        names = os.listdir(src)
        nret2 = 0
        for name in names:
            srcname = os.path.join(src, name)
            dstname = os.path.join(dst, name)
            if not name.startswith('arch') and os.path.isdir(
                    srcname) and os.path.isfile(
                        os.path.join(srcname, 'makefile')):
                os.mkdir(dstname)
                nret = self.copyExamples(srcname, dstname)
                if name == 'tests' or name == 'tutorials' or name == 'nlevp':
                    self.copyexamplefiles(srcname, dstname)
                    if os.path.isdir(os.path.join(srcname, 'output')):
                        os.mkdir(os.path.join(dstname, 'output'))
                        self.copyexamplefiles(os.path.join(srcname, 'output'),
                                              os.path.join(dstname, 'output'))
                    nret = 1
                if not nret:
                    # prune directory branches that don't have examples under them
                    os.unlink(os.path.join(dstname, 'makefile'))
                    os.rmdir(dstname)
                nret2 = nret + nret2
        return nret2
Example #17
0
def copytree(src, dst, symlinks=False, hardlinks=False, ignore=None):
    names = os.listdir(src)
    if ignore is not None:
        ignored_names = ignore(src, names)
    else:
        ignored_names = set()

    mkdirs(dst)

    errors = []
    for name in names:
        if name in ignored_names:
            continue
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                copytree(srcname, dstname, symlinks, hardlinks, ignore)
            elif hardlinks:
                os.link(srcname, dstname)
            else:
                shutil.copy2(srcname, dstname)
            # XXX What about devices, sockets etc.?
        except (IOError, os.error) as why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])

    if errors:
        raise shutil.Error(errors)
Example #18
0
 def copytree_rec(src, dst):
     names = os.listdir(src)
     os.mkdir(dst)
     errors = []
     for name in names:
         srcname = os.path.join(src, name)
         if srcname in skip:
             continue
         dstname = os.path.join(dst, name)
         try:
             if symlinks and os.path.islink(srcname):
                 linkto = os.readlink(srcname)
                 os.symlink(linkto, dstname)
             elif os.path.isdir(srcname):
                 copytree_rec(srcname, dstname)
             else:
                 shutil.copy2(srcname, dstname)
             # XXX What about devices, sockets etc.?
         except (IOError, OSError) as why:
             errors.append((srcname, dstname, str(why)))
         # catch the Error from the recursive copytree so that we can
         # continue with other files
         except shutil.Error as err:
             errors.extend(err.args[0])
     try:
         shutil.copystat(src, dst)
     except OSError as why:
         errors.append((src, dst, str(why)))
     if errors:
         raise shutil.Error(errors)
Example #19
0
def remove_obsolete_files(src, dst, ignore=None):
    names = os.listdir(dst)
    if ignore is not None:
        ignored_names = ignore(dst, names)
    else:
        ignored_names = set()

    errors = []
    for name in names:
        if name in ignored_names:
            continue
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if os.path.isdir(dstname):
                remove_obsolete_files(srcname, dstname, ignore)
            if not os.path.exists(srcname):
                print('remove %s' % dstname)
                if os.path.isfile(dstname):
                    os.unlink(dstname)
                else:
                    rmdir_p(dstname)
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])
        except OSError as why:
            errors.append((srcname, dstname, str(why)))
    if errors:
        raise shutil.Error(errors)
    return dst
def copy_file(src, dst, buffer_size=10485760, preserve_file_date=True):
    """
    Copies a file to a new location.
    Much faster performance than Apache Commons due to use of larger buffer.

    :param src:    Source file path
    :param dst:    Destination file path
    :param buffer_size:    Buffer size to use during copy
    :param preserve_file_date:    Preserve the original file date
    """

    # Optimize the buffer for small files
    buffer_size = min(buffer_size, os.path.getsize(src))
    if buffer_size == 0:
        buffer_size = 1024

    if shutil._samefile(src, dst):
        raise shutil.Error("`{0}` and `{1}` are the same file".format(
            src, dst))
    for fn in [src, dst]:
        try:
            st = os.stat(fn)
        except OSError:  # File most likely does not exist
            pass
        else:  # XXX What about other special files? (sockets, devices...)
            if shutil.stat.S_ISFIFO(st.st_mode):
                raise shutil.SpecialFileError(
                    "`{}` is a named pipe".format(fn))
    with open(src, 'rb') as fsrc:
        with open(dst, 'wb') as fdst:
            shutil.copyfileobj(fsrc, fdst, buffer_size)

    if preserve_file_date:
        shutil.copystat(src, dst)
Example #21
0
def main():
    args = getArgs()
    # Input directory
    inp = args.input if (checkDirExists(args.input, "input")) else sysExit()
    out = args.output if (checkDirExists(args.output, "output")) else sysExit()
    totalSamples = getDirSampleSize(inp)
    checkDirSampleSize(totalSamples, inp)
    # Get list of files in directory
    allSamples = remove_even(os.listdir(inp))
    # Make a copy of files in a new directory
    errors = []
    copied = 0
    sampleNum = 0
    print("Copying process started, please wait")
    for i in allSamples:
        filePath = os.path.join(inp, allSamples[sampleNum])
        #print(filePath+"\n")
        try:
            shutil.copy(filePath, out)
            copied += 1
        except OSError as why:
            errors.append((filePath, out + allSamples[i], str(why)))
        except shutil.Error as err:
            errors.extend(err.args[0])
        sampleNum += 1
    print("Sample copying complete (" + str(copied) + "/" +
          str(len(allSamples)) + " samples successful)")
    try:
        shutil.copystat(inp, out)
    except OSError as why:
        # can't copy file access times on Windows
        if why.winerror is None:
            errors.extend((inp, out, str(why)))
    if errors:
        raise shutil.Error(errors)
Example #22
0
def addtree(src, dst, symlinks=False):
    names = os.listdir(src)
    mkdirp(dst)
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                addtree(srcname, dstname, symlinks)
            else:
                shutil.copy2(srcname, dstname)
            # XXX What about devices, sockets etc.?
        except OSError as why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])
    try:
        shutil.copystat(src, dst)
    except OSError as why:
        # can't copy file access times on Windows
        if why.winerror is None:
            errors.extend((src, dst, str(why)))
    if errors:
        raise shutil.Error(errors)
Example #23
0
def copytree(src, dst, symlinks=False, exist_ok=False):
    names = os.listdir(src)

    os.makedirs(dst, exist_ok=exist_ok)
    errors = []
    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if os.path.islink(srcname):
                linkto = os.readlink(srcname)
                if os.path.isdir(srcname):
                    copytree(srcname, dstname, symlinks, exist_ok)
                else:
                    shutil.copy2(srcname, dstname)
            elif os.path.isdir(srcname):
                copytree(srcname, dstname, symlinks, exist_ok)
            else:
                # Will raise a SpecialFileError for unsupported file types
                shutil.copy2(srcname, dstname)
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:  # pragma: nocover
            errors.extend(err.args[0])
        except OSError as why:  # pragma: nocover
            errors.append((srcname, dstname, str(why)))
    try:
        shutil.copystat(src, dst)
    except OSError as why:  # pragma: nocover
        # Copying file access times may fail on Windows
        if getattr(why, 'winerror', None) is None:
            errors.append((src, dst, str(why)))
    if errors:  # pragma: nocover
        raise shutil.Error(errors)
    return dst
Example #24
0
 def copy_files(self, src, dst, symlinks=False, ignore=None,
                copy_function=shutil.copy2, ignore_dangling_symlinks=False):
     '''Copied from shutil.copytree, but `os.makedirs(dst)` is not called,
     so it assumes that `dst` exists.
     '''
     names = os.listdir(src)
     if ignore is not None:
         ignored_names = ignore(src, names)
     else:
         ignored_names = set()
     errors = []
     for name in names:
         if name in ignored_names:
             continue
         srcname = os.path.join(src, name)
         dstname = os.path.join(dst, name)
         try:
             if os.path.islink(srcname):
                 linkto = os.readlink(srcname)
                 if symlinks:
                     # We can't just leave it to `copy_function` because
                     # legacy code with a custom `copy_function` may rely on
                     # copytree doing the right thing.
                     os.symlink(linkto, dstname)
                     shutil.copystat(srcname, dstname,
                                     follow_symlinks=not symlinks)
                 else:
                     # ignore dangling symlink if the flag is on
                     if not os.path.exists(
                             linkto) and ignore_dangling_symlinks:
                         continue
                     # otherwise let the copy occurs. copy2 will raise
                     # an error
                     if os.path.isdir(srcname):
                         shutil.copytree(srcname, dstname, symlinks, ignore,
                                  copy_function)
                     else:
                         copy_function(srcname, dstname)
             elif os.path.isdir(srcname):
                 shutil.copytree(srcname, dstname, symlinks, ignore,
                                 copy_function)
             else:
                 # Will raise a SpecialFileError for unsupported file types
                 copy_function(srcname, dstname)
         # catch the Error from the recursive copytree so that we can
         # continue with other files
         except shutil.Error as err:
             errors.extend(err.args[0])
         except OSError as why:
             errors.append((srcname, dstname, str(why)))
     try:
         shutil.copystat(src, dst)
     except OSError as why:
         # Copying file access times may fail on Windows
         if getattr(why, 'winerror', None) is None:
             errors.append((src, dst, str(why)))
     if errors:
         raise shutil.Error(errors)
     return dst
Example #25
0
    def _setup_test_migrate_disk_and_power_off_mocks(self,
                                                     same_host=False,
                                                     with_exception=False):
        self._instance_data = self._get_instance_data()
        instance = db.instance_create(self._context, self._instance_data)
        network_info = fake_network.fake_get_instance_nw_info(self.stubs,
                                                              spectacular=True)

        fake_local_ip = '10.0.0.1'
        if same_host:
            fake_dest_ip = fake_local_ip
        else:
            fake_dest_ip = '10.0.0.2'

        fake_root_vhd_path = 'C:\\FakePath\\root.vhd'
        fake_revert_path = os.path.join(self._test_instance_dir, '_revert')

        func = mox.Func(self._check_instance_name)
        vmutils.VMUtils.set_vm_state(func, constants.HYPERV_VM_STATE_DISABLED)

        m = vmutils.VMUtils.get_vm_storage_paths(func)
        m.AndReturn(([fake_root_vhd_path], []))

        m = hostutils.HostUtils.get_local_ips()
        m.AndReturn([fake_local_ip])

        m = fake.PathUtils.get_instance_dir(mox.IsA(str))
        m.AndReturn(self._test_instance_dir)

        m = pathutils.PathUtils.get_instance_migr_revert_dir(instance['name'],
                                                             remove_dir=True)
        m.AndReturn(fake_revert_path)

        if same_host:
            fake.PathUtils.makedirs(mox.IsA(str))

        m = fake.PathUtils.copy(fake_root_vhd_path, mox.IsA(str))
        if with_exception:
            m.AndRaise(shutil.Error('Simulated copy error'))
            m = fake.PathUtils.get_instance_dir(mox.IsA(str),
                                                mox.IsA(str),
                                                remove_dir=True)
            m.AndReturn(self._test_instance_dir)
        else:
            fake.PathUtils.rename(mox.IsA(str), mox.IsA(str))
            destroy_disks = True
            if same_host:
                fake.PathUtils.rename(mox.IsA(str), mox.IsA(str))
                destroy_disks = False

            self._setup_destroy_mocks(False)

            if destroy_disks:
                m = fake.PathUtils.get_instance_dir(mox.IsA(str),
                                                    mox.IsA(str),
                                                    remove_dir=True)
                m.AndReturn(self._test_instance_dir)

        return (instance, fake_dest_ip, network_info)
Example #26
0
 def test_move_results_error(self):
     """Test that errors during the move other than the file being
     already present are raised
     """
     with mock.patch('os.listdir', side_effect=[['collection'], ['file']]), \
             mock.patch('os.makedirs'), \
             mock.patch('shutil.move', side_effect=shutil.Error('some error')):
         with self.assertRaises(shutil.Error):
             self.converter.move_results('foo', 'bar')
Example #27
0
    def copytreeFilter(self,
                       dst,
                       symlinks=False,
                       filterDir=None,
                       filterFile=None):
        """Recursively copy a directory tree using copy2().

        The destination directory must not already exist.
        If exception(s) occur, an Error is raised with a list of reasons.

        If the optional symlinks flag is true, symbolic links in the
        source tree result in symbolic links in the destination tree; if
        it is false, the contents of the files pointed to by symbolic
        links are copied.

        XXX Consider this example code rather than the ultimate tool.

        'filterDir' will be called passing each source directory name, if it returns False, the directory will not be copied.
        'filterFile' will be called passing each source file name, if it returns False, the file will not be copied.
        """
        dst = Path(dst)
        names = self.listdir()
        dst.mkdir()
        errors = []
        for name in names:
            srcname = self / self.relpathto(name)
            dstname = dst / self.relpathto(name)
            try:
                if symlinks and os.path.islink(srcname):
                    linkto = os.readlink(srcname)
                    os.symlink(linkto, dstname)
                elif os.path.isdir(srcname):
                    if filterDir is not None:
                        if not filterDir(srcname):
                            continue
                    srcname.copytreeFilter(dstname, symlinks, filterDir,
                                           filterFile)
                else:
                    if filterFile is not None:
                        if not filterFile(srcname):
                            continue
                    srcname.copy2(dstname)
                # XXX What about devices, sockets etc.?
            except (IOError, os.error) as why:
                import pdb
                pdb.set_trace()
                errors.append((srcname, dstname, why))
            # catch the Error from the recursive copytree so that we can
            # continue with other files
            except shutil.Error as err:
                import pdb
                pdb.set_trace()
                errors.extend(err.args[0])
        if errors:
            import pdb
            pdb.set_trace()
            raise shutil.Error(errors)
Example #28
0
    def copy_tree(self, src: Path, dst: Path):
        """ based on shutil.copytree
        """
        self.last_src, self.last_dst = os.fspath(src), os.fspath(dst)
        save_top_destination_does_not_exist = self.top_destination_does_not_exist
        self.top_destination_does_not_exist = self.top_destination_does_not_exist or not dst.exists()  # !

        self.doing = f"""copy folder '{self.last_src}' to '{self.last_dst}'"""

        src_dir_items = list(os.scandir(src))
        src_file_names = [src_item.name for src_item in src_dir_items if src_item.is_file()]
        src_dir_names = [src_item.name for src_item in src_dir_items if src_item.is_dir()]
        if not self.should_copy_dir(src, dst, src_file_names):
            self.statistics['skipped_dirs'] += 1
            return

        self.statistics['dirs'] += 1
        log.debug(f"copy folder '{src}' to '{dst}'")

        # call MakeDir even if dst already exists, so permissions/ACL (and possibly owner) will be set correctly
        with MakeDir(dst, chowner=(self.copy_owner and self.has_chown), own_progress_count=0) as dir_maker:
            dir_maker()

        if not self.top_destination_does_not_exist and self.delete_extraneous_files:
            self.remove_extraneous_files(dst, src_file_names+src_dir_names)

        errors = []
        for src_item in src_dir_items:
            src_item_path = Path(src_item.path)
            if self.should_ignore_file(src_item_path):
                self.statistics['ignored'] += 1
                continue
            dst_path = dst.joinpath(src_item.name)
            try:
                if src_item.is_symlink():
                    self.statistics['symlinks'] += 1
                    self.copy_symlink(src_item_path, dst_path)
                elif src_item.is_dir():
                    self.copy_tree(src_item_path, dst_path)
                else:
                    self.statistics['files'] += 1
                    # Will raise a SpecialFileError for unsupported file types
                    self.copy_file_to_file_DirEntry(src_item, dst_path)
            # catch the Error from the recursive copytree so that we can
            # continue with other files
            except shutil.Error as err:
                errors.append(err.args[0])
            except OSError as why:
                errors.append((src_item_path, dst_path, str(why)))

        if errors:
            raise shutil.Error(errors)

        self.top_destination_does_not_exist = save_top_destination_does_not_exist
        return dst
Example #29
0
def copytree(src, dst):
    """
    Copy recursively the `src` directory to the `dst` directory. If `dst` is an
    existing directory, files in `dst` may be overwritten during the copy.
    Preserve timestamps.
    Ignores:
     -`src` permissions: `dst` files are created with the default permissions.
     - all special files such as FIFO or character devices and symlinks.

    Raise an shutil.Error with a list of reasons.

    This function is similar to and derived from the Python shutil.copytree
    function. See fileutils.py.ABOUT for details.
    """
    if on_linux and py2:
        src = fsencode(src)
        dst = fsencode(dst)

    if not filetype.is_readable(src):
        chmod(src, R, recurse=False)

    names = os.listdir(src)

    if not os.path.exists(dst):
        os.makedirs(dst)

    errors = []
    errors.extend(copytime(src, dst))

    for name in names:
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)

        # skip anything that is not a regular file, dir or link
        if not filetype.is_regular(srcname):
            continue

        if not filetype.is_readable(srcname):
            chmod(srcname, R, recurse=False)
        try:
            if os.path.isdir(srcname):
                copytree(srcname, dstname)
            elif filetype.is_file(srcname):
                copyfile(srcname, dstname)
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])
        except EnvironmentError as why:
            errors.append((srcname, dstname, str(why)))

    if errors:
        raise shutil.Error(errors)
Example #30
0
def copytree(src, dst, symlinks=False, ignore=None, hardlinks=False):
    '''copytree that supports hard-linking
    '''
    names = os.listdir(src)
    if ignore is not None:
        ignored_names = ignore(src, names)
    else:
        ignored_names = set()

    if hardlinks:

        def copy(srcname, dstname):
            try:
                # try hard-linking first
                os.link(srcname, dstname)
            except OSError:
                shutil.copy2(srcname, dstname)
    else:
        copy = shutil.copy2

    if not os.path.isdir(dst):
        os.makedirs(dst)

    errors = []
    for name in names:
        if name in ignored_names:
            continue
        srcname = os.path.join(src, name)
        dstname = os.path.join(dst, name)
        try:
            if symlinks and os.path.islink(srcname):
                linkto = os.readlink(srcname)
                os.symlink(linkto, dstname)
            elif os.path.isdir(srcname):
                copytree(srcname, dstname, symlinks, ignore)
            else:
                copy(srcname, dstname)
        # XXX What about devices, sockets etc.?
        except (IOError, os.error) as why:
            errors.append((srcname, dstname, str(why)))
        # catch the Error from the recursive copytree so that we can
        # continue with other files
        except shutil.Error as err:
            errors.extend(err.args[0])
    try:
        shutil.copystat(src, dst)
    except shutil.WindowsError:
        # can't copy file access times on Windows
        pass
    except OSError as why:
        errors.extend((src, dst, str(why)))
    if errors:
        raise shutil.Error(errors)