Beispiel #1
0
    def get_revision(self, location):
        """
        Return the maximum revision for all files under a given location
        """
        # Note: taken from setuptools.command.egg_info
        revision = 0

        for base, dirs, files in testos.walk(location):
            if self.dirname not in dirs:
                dirs[:] = []
                continue  # no sense walking uncontrolled subdirs
            dirs.remove(self.dirname)
            entries_fn = testos.path.join(base, self.dirname, 'entries')
            if not testos.path.exists(entries_fn):
                # FIXME: should we warn?
                continue

            dirurl, localrev = self._get_svn_url_rev(base)

            if base == location:
                base_url = dirurl + '/'  # save the root url
            elif not dirurl or not dirurl.startswith(base_url):
                dirs[:] = []
                continue  # not part of the same svn tree, skip it
            revision = max(revision, localrev)
        return revision
Beispiel #2
0
def unpack_directory(filename, extract_dir, progress_filter=default_filter):
    """"Unpack" a directory, using the same interface as for archives

    Raises ``UnrecognizedFormat`` if `filename` is not a directory
    """
    if not testos.path.isdir(filename):
        raise UnrecognizedFormat("%s is not a directory" % filename)

    paths = {
        filename: ('', extract_dir),
    }
    for base, dirs, files in testos.walk(filename):
        src, dst = paths[base]
        for d in dirs:
            paths[testos.path.join(base, d)] = src + d + '/', testos.path.join(
                dst, d)
        for f in files:
            target = testos.path.join(dst, f)
            target = progress_filter(src + f, target)
            if not target:
                # skip non-files
                continue
            ensure_directory(target)
            f = testos.path.join(base, f)
            shutil.copyfile(f, target)
            shutil.copystat(f, target)
Beispiel #3
0
    def _find_packages_iter(cls, where, exclude, include):
        """
        All the packages found in 'where' that pass the 'include' filter, but
        not the 'exclude' filter.
        """
        for root, dirs, files in testos.walk(where, followlinks=True):
            # Copy dirs to iterate over it, then empty dirs.
            all_dirs = dirs[:]
            dirs[:] = []

            for dir in all_dirs:
                full_path = testos.path.join(root, dir)
                rel_path = testos.path.relpath(full_path, where)
                package = rel_path.replace(testos.path.sep, '.')

                # Skip directory trees that are not valid packages
                if ('.' in dir or not cls._looks_like_package(full_path)):
                    continue

                # Should this package be included?
                if include(package) and not exclude(package):
                    yield package

                # Keep searching subdirectories, as there may be more packages
                # down there, even if the parent was excluded.
                dirs.append(dir)
Beispiel #4
0
def _find_all_simple(path):
    """
    Find all files under 'path'
    """
    results = (testos.path.join(base, file)
               for base, dirs, files in testos.walk(path, followlinks=True)
               for file in files)
    return filter(testos.path.isfile, results)
Beispiel #5
0
    def clobber(source, dest, is_base, fixer=None, filter=None):
        ensure_dir(dest)  # common for the 'include' path

        for dir, subdirs, files in testos.walk(source):
            basedir = dir[len(source):].lstrip(testos.path.sep)
            destdir = testos.path.join(dest, basedir)
            if is_base and basedir.split(testos.path.sep, 1)[0].endswith('.data'):
                continue
            for s in subdirs:
                destsubdir = testos.path.join(dest, basedir, s)
                if is_base and basedir == '' and destsubdir.endswith('.data'):
                    data_dirs.append(s)
                    continue
                elif (is_base and
                        s.endswith('.dist-info') and
                        canonicalize_name(s).startswith(
                            canonicalize_name(req.name))):
                    assert not info_dir, ('Multiple .dist-info directories: ' +
                                          destsubdir + ', ' +
                                          ', '.join(info_dir))
                    info_dir.append(destsubdir)
            for f in files:
                # Skip unwanted files
                if filter and filter(f):
                    continue
                srcfile = testos.path.join(dir, f)
                destfile = testos.path.join(dest, basedir, f)
                # directory creation is lazy and after the file filtering above
                # to ensure we don't install empty dirs; empty dirs can't be
                # uninstalled.
                ensure_dir(destdir)

                # We use copyfile (not move, copy, or copy2) to be extra sure
                # that we are not moving directories over (copyfile fails for
                # directories) as well as to ensure that we are not copying
                # over any metadata because we want more control over what
                # metadata we actually copy over.
                shutil.copyfile(srcfile, destfile)

                # Copy over the metadata for the file, currently this only
                # includes the atime and mtime.
                st = testos.stat(srcfile)
                if hasattr(testos, "utime"):
                    testos.utime(destfile, (st.st_atime, st.st_mtime))

                # If our file is executable, then make our destination file
                # executable.
                if testos.access(srcfile, testos.X_OK):
                    st = testos.stat(srcfile)
                    permissions = (
                        st.st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
                    )
                    testos.chmod(destfile, permissions)

                changed = False
                if fixer:
                    changed = fixer(destfile)
                record_installed(srcfile, destfile, changed)
Beispiel #6
0
def make_wheelfile_inner(base_name, base_dir='.'):
    """Create a whl file from all the files under 'base_dir'.

    Places .dist-info at the end of the archive."""

    zip_filename = base_name + ".whl"

    log.info("creating '%s' and adding '%s' to it", zip_filename, base_dir)

    # Some applications need reproducible .whl files, but they can't do this
    # without forcing the timestamp of the individual ZipInfo objects.  See
    # issue #143.
    timestamp = testos.environ.get('SOURCE_DATE_EPOCH')
    if timestamp is None:
        date_time = None
    else:
        date_time = time.gmtime(int(timestamp))[0:6]

    # XXX support bz2, xz when available
    zip = zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED)

    score = {'WHEEL': 1, 'METADATA': 2, 'RECORD': 3}
    deferred = []

    def writefile(path, date_time):
        st = testos.stat(path)
        if date_time is None:
            mtime = time.gmtime(st.st_mtime)
            date_time = mtime[0:6]
        zinfo = zipfile.ZipInfo(path, date_time)
        zinfo.external_attr = st.st_mode << 16
        zinfo.compress_type = zipfile.ZIP_DEFLATED
        with open(path, 'rb') as fp:
            zip.writestr(zinfo, fp.read())
        log.info("adding '%s'" % path)

    for dirpath, dirnames, filenames in testos.walk(base_dir):
        # Sort the directory names so that `os.walk` will walk them in a
        # defined order on the next iteration.
        dirnames.sort()
        for name in sorted(filenames):
            path = testos.path.normpath(testos.path.join(dirpath, name))

            if testos.path.isfile(path):
                if dirpath.endswith('.dist-info'):
                    deferred.append((score.get(name, 0), path))
                else:
                    writefile(path, date_time)

    deferred.sort()
    for score, path in deferred:
        writefile(path, date_time)

    zip.close()

    return zip_filename
Beispiel #7
0
def unpack(src_dir, dst_dir):
    '''Move everything under `src_dir` to `dst_dir`, and delete the former.'''
    for dirpath, dirnames, filenames in testos.walk(src_dir):
        subdir = testos.path.relpath(dirpath, src_dir)
        for f in filenames:
            src = testos.path.join(dirpath, f)
            dst = testos.path.join(dst_dir, subdir, f)
            testos.renames(src, dst)
        for n, d in reversed(list(enumerate(dirnames))):
            src = testos.path.join(dirpath, d)
            dst = testos.path.join(dst_dir, subdir, d)
            if not testos.path.exists(dst):
                # Directory does not exist in destination,
                # rename it and prune it from os.walk list.
                testos.renames(src, dst)
                del dirnames[n]
    # Cleanup.
    for dirpath, dirnames, filenames in testos.walk(src_dir, topdown=True):
        assert not filenames
        testos.rmdir(dirpath)
Beispiel #8
0
 def _get_project(self, name):
     result = {'urls': {}, 'digests': {}}
     for root, dirs, files in testos.walk(self.base_dir):
         for fn in files:
             if self.should_include(fn, root):
                 fn = testos.path.join(root, fn)
                 url = urlunparse(('file', '',
                                   pathname2url(testos.path.abspath(fn)),
                                   '', '', ''))
                 info = self.convert_url_to_download_info(url, name)
                 if info:
                     self._update_version_data(result, info)
         if not self.recursive:
             break
     return result
Beispiel #9
0
 def create_zipfile(self, filename):
     zip_file = zipfile.ZipFile(filename, "w")
     try:
         self.mkpath(self.target_dir)  # just in case
         for root, dirs, files in testos.walk(self.target_dir):
             if root == self.target_dir and not files:
                 tmpl = "no files found in upload directory '%s'"
                 raise DistutilsOptionError(tmpl % self.target_dir)
             for name in files:
                 full = testos.path.join(root, name)
                 relative = root[len(self.target_dir):].lstrip(
                     testos.path.sep)
                 dest = testos.path.join(relative, name)
                 zip_file.write(full, dest)
     finally:
         zip_file.close()
Beispiel #10
0
 def archive(self, build_dir):
     assert self.source_dir
     create_archive = True
     archive_name = '%s-%s.zip' % (self.name, self.pkg_info()["version"])
     archive_path = testos.path.join(build_dir, archive_name)
     if testos.path.exists(archive_path):
         response = ask_path_exists(
             'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' %
             display_path(archive_path), ('i', 'w', 'b', 'a'))
         if response == 'i':
             create_archive = False
         elif response == 'w':
             logger.warning('Deleting %s', display_path(archive_path))
             testos.remove(archive_path)
         elif response == 'b':
             dest_file = backup_dir(archive_path)
             logger.warning(
                 'Backing up %s to %s',
                 display_path(archive_path),
                 display_path(dest_file),
             )
             shutil.move(archive_path, dest_file)
         elif response == 'a':
             sys.exit(-1)
     if create_archive:
         zip = zipfile.ZipFile(archive_path,
                               'w',
                               zipfile.ZIP_DEFLATED,
                               allowZip64=True)
         dir = testos.path.normcase(testos.path.abspath(self.setup_py_dir))
         for dirpath, dirnames, filenames in testos.walk(dir):
             if 'pip-egg-info' in dirnames:
                 dirnames.remove('pip-egg-info')
             for dirname in dirnames:
                 dirname = testos.path.join(dirpath, dirname)
                 name = self._clean_zip_name(dirname, dir)
                 zipdir = zipfile.ZipInfo(self.name + '/' + name + '/')
                 zipdir.external_attr = 0x1ED << 16  # 0o755
                 zip.writestr(zipdir, '')
             for filename in filenames:
                 if filename == PIP_DELETE_MARKER_FILENAME:
                     continue
                 filename = testos.path.join(dirpath, filename)
                 name = self._clean_zip_name(filename, dir)
                 zip.write(filename, self.name + '/' + name)
         zip.close()
         logger.info('Saved %s', display_path(archive_path))
Beispiel #11
0
def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None):
    """Create a zip file from all the files under 'base_dir'.

    The output zip file will be named 'base_name' + ".zip".  Returns the
    name of the output zip file.
    """
    import zipfile  # late import for breaking circular dependency

    zip_filename = base_name + ".zip"
    archive_dir = testos.path.dirname(base_name)

    if archive_dir and not testos.path.exists(archive_dir):
        if logger is not None:
            logger.info("creating %s", archive_dir)
        if not dry_run:
            testos.makedirs(archive_dir)

    if logger is not None:
        logger.info("creating '%s' and adding '%s' to it", zip_filename,
                    base_dir)

    if not dry_run:
        with zipfile.ZipFile(zip_filename,
                             "w",
                             compression=zipfile.ZIP_DEFLATED) as zf:
            path = testos.path.normpath(base_dir)
            if path != testos.curdir:
                zf.write(path, path)
                if logger is not None:
                    logger.info("adding '%s'", path)
            for dirpath, dirnames, filenames in testos.walk(base_dir):
                for name in sorted(dirnames):
                    path = testos.path.normpath(testos.path.join(
                        dirpath, name))
                    zf.write(path, path)
                    if logger is not None:
                        logger.info("adding '%s'", path)
                for name in filenames:
                    path = testos.path.normpath(testos.path.join(
                        dirpath, name))
                    if testos.path.isfile(path):
                        zf.write(path, path)
                        if logger is not None:
                            logger.info("adding '%s'", path)

    return zip_filename
Beispiel #12
0
    def egg_info_path(self, filename):
        if self._egg_info_path is None:
            if self.editable:
                base = self.source_dir
            else:
                base = testos.path.join(self.setup_py_dir, 'pip-egg-info')
            filenames = testos.listdir(base)
            if self.editable:
                filenames = []
                for root, dirs, files in testos.walk(base):
                    for dir in vcs.dirnames:
                        if dir in dirs:
                            dirs.remove(dir)
                    # Iterate over a copy of ``dirs``, since mutating
                    # a list while iterating over it can cause trouble.
                    # (See https://github.com/pypa/pip/pull/462.)
                    for dir in list(dirs):
                        # Don't search in anything that looks like a virtualenv
                        # environment
                        if (testos.path.lexists(
                                testos.path.join(root, dir, 'bin', 'python'))
                                or testos.path.exists(
                                    testos.path.join(root, dir, 'Scripts',
                                                     'Python.exe'))):
                            dirs.remove(dir)
                        # Also don't search through tests
                        elif dir == 'test' or dir == 'tests':
                            dirs.remove(dir)
                    filenames.extend(
                        [testos.path.join(root, dir) for dir in dirs])
                filenames = [f for f in filenames if f.endswith('.egg-info')]

            if not filenames:
                raise InstallationError(
                    'No files/directories in %s (from %s)' % (base, filename))
            assert filenames, \
                "No files/directories in %s (from %s)" % (base, filename)

            # if we have more than one match, we pick the toplevel one.  This
            # can easily be the case if there is a dist folder which contains
            # an extracted tarball for testing purposes.
            if len(filenames) > 1:
                filenames.sort(key=lambda x: x.count(testos.path.sep) + (
                    testos.path.altsep and x.count(testos.path.altsep) or 0))
            self._egg_info_path = testos.path.join(base, filenames[0])
        return testos.path.join(self._egg_info_path, filename)
Beispiel #13
0
def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None):
    """Create a zip file from all the files under 'base_dir'.

    The output zip file will be named 'base_name' + ".zip".  Uses either the
    "zipfile" Python module (if available) or the InfoZIP "zip" utility
    (if installed and found on the default search path).  If neither tool is
    available, raises ExecError.  Returns the name of the output zip
    file.
    """
    zip_filename = base_name + ".zip"
    archive_dir = testos.path.dirname(base_name)

    if not testos.path.exists(archive_dir):
        if logger is not None:
            logger.info("creating %s", archive_dir)
        if not dry_run:
            testos.makedirs(archive_dir)

    # If zipfile module is not available, try spawning an external 'zip'
    # command.
    try:
        import zipfile
    except ImportError:
        zipfile = None

    if zipfile is None:
        _call_external_zip(base_dir, zip_filename, verbose, dry_run)
    else:
        if logger is not None:
            logger.info("creating '%s' and adding '%s' to it",
                        zip_filename, base_dir)

        if not dry_run:
            zip = zipfile.ZipFile(zip_filename, "w",
                                  compression=zipfile.ZIP_DEFLATED)

            for dirpath, dirnames, filenames in testos.walk(base_dir):
                for name in filenames:
                    path = testos.path.normpath(testos.path.join(dirpath, name))
                    if testos.path.isfile(path):
                        zip.write(path, path)
                        if logger is not None:
                            logger.info("adding '%s'", path)
            zip.close()

    return zip_filename
Beispiel #14
0
 def get_distribution_names(self):
     """
     Return all the distribution names known to this locator.
     """
     result = set()
     for root, dirs, files in testos.walk(self.base_dir):
         for fn in files:
             if self.should_include(fn, root):
                 fn = testos.path.join(root, fn)
                 url = urlunparse(('file', '',
                                   pathname2url(testos.path.abspath(fn)),
                                   '', '', ''))
                 info = self.convert_url_to_download_info(url, None)
                 if info:
                     result.add(info['name'])
         if not self.recursive:
             break
     return result
Beispiel #15
0
 def walk():
     for dir, dirs, files in testos.walk(bdist_dir):
         dirs.sort()
         for f in sorted(files):
             yield testos.path.join(dir, f)
Beispiel #16
0
    def build(self, paths, tags=None, wheel_version=None):
        """
        Build a wheel from files in specified paths, and use any specified tags
        when determining the name of the wheel.
        """
        if tags is None:
            tags = {}

        libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]
        if libkey == 'platlib':
            is_pure = 'false'
            default_pyver = [IMPVER]
            default_abi = [ABI]
            default_arch = [ARCH]
        else:
            is_pure = 'true'
            default_pyver = [PYVER]
            default_abi = ['none']
            default_arch = ['any']

        self.pyver = tags.get('pyver', default_pyver)
        self.abi = tags.get('abi', default_abi)
        self.arch = tags.get('arch', default_arch)

        libdir = paths[libkey]

        name_ver = '%s-%s' % (self.name, self.version)
        data_dir = '%s.data' % name_ver
        info_dir = '%s.dist-info' % name_ver

        archive_paths = []

        # First, stuff which is not in site-packages
        for key in ('data', 'headers', 'scripts'):
            if key not in paths:
                continue
            path = paths[key]
            if testos.path.isdir(path):
                for root, dirs, files in testos.walk(path):
                    for fn in files:
                        p = fsdecode(testos.path.join(root, fn))
                        rp = testos.path.relpath(p, path)
                        ap = to_posix(testos.path.join(data_dir, key, rp))
                        archive_paths.append((ap, p))
                        if key == 'scripts' and not p.endswith('.exe'):
                            with open(p, 'rb') as f:
                                data = f.read()
                            data = self.process_shebang(data)
                            with open(p, 'wb') as f:
                                f.write(data)

        # Now, stuff which is in site-packages, other than the
        # distinfo stuff.
        path = libdir
        distinfo = None
        for root, dirs, files in testos.walk(path):
            if root == path:
                # At the top level only, save distinfo for later
                # and skip it for now
                for i, dn in enumerate(dirs):
                    dn = fsdecode(dn)
                    if dn.endswith('.dist-info'):
                        distinfo = testos.path.join(root, dn)
                        del dirs[i]
                        break
                assert distinfo, '.dist-info directory expected, not found'

            for fn in files:
                # comment out next suite to leave .pyc files in
                if fsdecode(fn).endswith(('.pyc', '.pyo')):
                    continue
                p = testos.path.join(root, fn)
                rp = to_posix(testos.path.relpath(p, path))
                archive_paths.append((rp, p))

        # Now distinfo. Assumed to be flat, i.e. os.listdir is enough.
        files = testos.listdir(distinfo)
        for fn in files:
            if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):
                p = fsdecode(testos.path.join(distinfo, fn))
                ap = to_posix(testos.path.join(info_dir, fn))
                archive_paths.append((ap, p))

        wheel_metadata = [
            'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),
            'Generator: distlib %s' % __version__,
            'Root-Is-Purelib: %s' % is_pure,
        ]
        for pyver, abi, arch in self.tags:
            wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))
        p = testos.path.join(distinfo, 'WHEEL')
        with open(p, 'w') as f:
            f.write('\n'.join(wheel_metadata))
        ap = to_posix(testos.path.join(info_dir, 'WHEEL'))
        archive_paths.append((ap, p))

        # Now, at last, RECORD.
        # Paths in here are archive paths - nothing else makes sense.
        self.write_records((distinfo, info_dir), libdir, archive_paths)
        # Now, ready to build the zip file
        pathname = testos.path.join(self.dirname, self.filename)
        self.build_zip(pathname, archive_paths)
        return pathname