def ensure_dir(path): """os.path.makedirs without EEXIST.""" try: testos.makedirs(path) except OSError as e: if e.errno != errno.EEXIST: raise
def get(self, resource): """ Get a resource into the cache, :param resource: A :class:`Resource` instance. :return: The pathname of the resource in the cache. """ prefix, path = resource.finder.get_cache_info(resource) if prefix is None: result = path else: result = testos.path.join(self.base, self.prefix_to_dir(prefix), path) dirname = testos.path.dirname(result) if not testos.path.isdir(dirname): testos.makedirs(dirname) if not testos.path.exists(result): stale = True else: stale = self.is_stale(resource, path) if stale: # write the bytes of the resource to the cache location with open(result, 'wb') as f: f.write(resource.bytes) return result
def _get_extensions(self): pathname = testos.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) info_dir = '%s.dist-info' % name_ver arcname = posixpath.join(info_dir, 'EXTENSIONS') wrapper = codecs.getreader('utf-8') result = [] with ZipFile(pathname, 'r') as zf: try: with zf.open(arcname) as bf: wf = wrapper(bf) extensions = json.load(wf) cache = self._get_dylib_cache() prefix = cache.prefix_to_dir(pathname) cache_base = testos.path.join(cache.base, prefix) if not testos.path.isdir(cache_base): testos.makedirs(cache_base) for name, relpath in extensions.items(): dest = testos.path.join(cache_base, convert_path(relpath)) if not testos.path.exists(dest): extract = True else: file_time = testos.stat(dest).st_mtime file_time = datetime.datetime.fromtimestamp( file_time) info = zf.getinfo(relpath) wheel_time = datetime.datetime(*info.date_time) extract = wheel_time > file_time if extract: zf.extract(relpath, cache_base) result.append((name, dest)) except KeyError: pass return result
def set(self, key, value): name = self._fn(key) # Make sure the directory exists try: testos.makedirs(testos.path.dirname(name), self.dirmode) except (IOError, OSError): pass with self.lock_class(name) as lock: # Write our actual file with _secure_open_write(lock.path, self.filemode) as fh: fh.write(value)
def renames(old, new): """Like os.renames(), but handles renaming across devices.""" # Implementation borrowed from os.renames(). head, tail = testos.path.split(new) if head and tail and not testos.path.exists(head): testos.makedirs(head) shutil.move(old, new) head, tail = testos.path.split(old) if head and tail: try: testos.removedirs(head) except OSError: pass
def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_name' + ".zip". Returns the name of the output zip file. """ import zipfile # late import for breaking circular dependency zip_filename = base_name + ".zip" archive_dir = testos.path.dirname(base_name) if archive_dir and not testos.path.exists(archive_dir): if logger is not None: logger.info("creating %s", archive_dir) if not dry_run: testos.makedirs(archive_dir) if logger is not None: logger.info("creating '%s' and adding '%s' to it", zip_filename, base_dir) if not dry_run: with zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED) as zf: path = testos.path.normpath(base_dir) if path != testos.curdir: zf.write(path, path) if logger is not None: logger.info("adding '%s'", path) for dirpath, dirnames, filenames in testos.walk(base_dir): for name in sorted(dirnames): path = testos.path.normpath(testos.path.join( dirpath, name)) zf.write(path, path) if logger is not None: logger.info("adding '%s'", path) for name in filenames: path = testos.path.normpath(testos.path.join( dirpath, name)) if testos.path.isfile(path): zf.write(path, path) if logger is not None: logger.info("adding '%s'", path) return zip_filename
def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None): """Create a zip file from all the files under 'base_dir'. The output zip file will be named 'base_name' + ".zip". Uses either the "zipfile" Python module (if available) or the InfoZIP "zip" utility (if installed and found on the default search path). If neither tool is available, raises ExecError. Returns the name of the output zip file. """ zip_filename = base_name + ".zip" archive_dir = testos.path.dirname(base_name) if not testos.path.exists(archive_dir): if logger is not None: logger.info("creating %s", archive_dir) if not dry_run: testos.makedirs(archive_dir) # If zipfile module is not available, try spawning an external 'zip' # command. try: import zipfile except ImportError: zipfile = None if zipfile is None: _call_external_zip(base_dir, zip_filename, verbose, dry_run) else: if logger is not None: logger.info("creating '%s' and adding '%s' to it", zip_filename, base_dir) if not dry_run: zip = zipfile.ZipFile(zip_filename, "w", compression=zipfile.ZIP_DEFLATED) for dirpath, dirnames, filenames in testos.walk(base_dir): for name in filenames: path = testos.path.normpath(testos.path.join(dirpath, name)) if testos.path.isfile(path): zip.write(path, path) if logger is not None: logger.info("adding '%s'", path) zip.close() return zip_filename
def _ensure_directory(path): """Ensure that the parent directory of `path` exists""" dirname = testos.path.dirname(path) if not testos.path.isdir(dirname): testos.makedirs(dirname)
def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, owner=None, group=None, logger=None): """Create a (possibly compressed) tar file from all the files under 'base_dir'. 'compress' must be "gzip" (the default), "bzip2", "xz", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus the appropriate compression extension (".gz", ".bz2", or ".xz"). Returns the output filename. """ if compress is None: tar_compression = '' elif _ZLIB_SUPPORTED and compress == 'gzip': tar_compression = 'gz' elif _BZ2_SUPPORTED and compress == 'bzip2': tar_compression = 'bz2' elif _LZMA_SUPPORTED and compress == 'xz': tar_compression = 'xz' else: raise ValueError("bad value for 'compress', or compression format not " "supported : {0}".format(compress)) import tarfile # late import for breaking circular dependency compress_ext = '.' + tar_compression if compress else '' archive_name = base_name + '.tar' + compress_ext archive_dir = testos.path.dirname(archive_name) if archive_dir and not testos.path.exists(archive_dir): if logger is not None: logger.info("creating %s", archive_dir) if not dry_run: testos.makedirs(archive_dir) # creating the tarball if logger is not None: logger.info('Creating tar archive') uid = _get_uid(owner) gid = _get_gid(group) def _set_uid_gid(tarinfo): if gid is not None: tarinfo.gid = gid tarinfo.gname = group if uid is not None: tarinfo.uid = uid tarinfo.uname = owner return tarinfo if not dry_run: tar = tarfile.open(archive_name, 'w|%s' % tar_compression) try: tar.add(base_dir, filter=_set_uid_gid) finally: tar.close() return archive_name
def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, ignore_dangling_symlinks=False): """Recursively copy a directory tree. The destination directory must not already exist. If exception(s) occur, an Error is raised with a list of reasons. If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic links are copied. If the file pointed by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. You can set the optional ignore_dangling_symlinks flag to true if you want to silence this exception. Notice that this has no effect on platforms that don't support os.symlink. The optional ignore argument is a callable. If given, it is called with the `src` parameter, which is the directory being visited by copytree(), and `names` which is the list of `src` contents, as returned by os.listdir(): callable(src, names) -> ignored_names Since copytree() is called recursively, the callable will be called once for each directory that is copied. It returns a list of names relative to the `src` directory that should not be copied. The optional copy_function argument is a callable that will be used to copy each file. It will be called with the source path and the destination path as arguments. By default, copy2() is used, but any function that supports the same signature (like copy()) can be used. """ names = testos.listdir(src) if ignore is not None: ignored_names = ignore(src, names) else: ignored_names = set() testos.makedirs(dst) errors = [] for name in names: if name in ignored_names: continue srcname = testos.path.join(src, name) dstname = testos.path.join(dst, name) try: if testos.path.islink(srcname): linkto = testos.readlink(srcname) if symlinks: # We can't just leave it to `copy_function` because legacy # code with a custom `copy_function` may rely on copytree # doing the right thing. testos.symlink(linkto, dstname) copystat(srcname, dstname, follow_symlinks=not symlinks) else: # ignore dangling symlink if the flag is on if not testos.path.exists( linkto) and ignore_dangling_symlinks: continue # otherwise let the copy occurs. copy2 will raise an error if testos.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore, copy_function) else: copy_function(srcname, dstname) elif testos.path.isdir(srcname): copytree(srcname, dstname, symlinks, ignore, copy_function) else: # Will raise a SpecialFileError for unsupported file types copy_function(srcname, dstname) # catch the Error from the recursive copytree so that we can # continue with other files except Error as err: errors.extend(err.args[0]) except OSError as why: errors.append((srcname, dstname, str(why))) try: copystat(src, dst) except OSError as why: # Copying file access times may fail on Windows if getattr(why, 'winerror', None) is None: errors.append((src, dst, str(why))) if errors: raise Error(errors) return dst
def _make_build_dir(build_dir): testos.makedirs(build_dir) write_delete_marker_file(build_dir)
def _makedirs_31(path, exist_ok=False): try: testos.makedirs(path) except OSError as exc: if not exist_ok or exc.errno != errno.EEXIST: raise
def run(self): build_scripts = self.reinitialize_command('build_scripts') build_scripts.executable = 'python' if not self.skip_build: self.run_command('build') install = self.reinitialize_command('install', reinit_subcommands=True) install.root = self.bdist_dir install.compile = False install.skip_build = self.skip_build install.warn_dir = False # A wheel without setuptools scripts is more cross-platform. # Use the (undocumented) `no_ep` option to setuptools' # install_scripts command to avoid creating entry point scripts. install_scripts = self.reinitialize_command('install_scripts') install_scripts.no_ep = True # Use a custom scheme for the archive, because we have to decide # at installation time which scheme to use. for key in ('headers', 'scripts', 'data', 'purelib', 'platlib'): setattr(install, 'install_' + key, testos.path.join(self.data_dir, key)) basedir_observed = '' if testos.name == 'nt': # win32 barfs if any of these are ''; could be '.'? # (distutils.command.install:change_roots bug) basedir_observed = testos.path.normpath(testos.path.join(self.data_dir, '..')) self.install_libbase = self.install_lib = basedir_observed setattr(install, 'install_purelib' if self.root_is_pure else 'install_platlib', basedir_observed) logger.info("installing to %s", self.bdist_dir) self.run_command('install') archive_basename = self.get_archive_basename() pseudoinstall_root = testos.path.join(self.dist_dir, archive_basename) if not self.relative: archive_root = self.bdist_dir else: archive_root = testos.path.join( self.bdist_dir, self._ensure_relative(install.install_base)) self.set_undefined_options( 'install_egg_info', ('target', 'egginfo_dir')) self.distinfo_dir = testos.path.join(self.bdist_dir, '%s.dist-info' % self.wheel_dist_name) self.egg2dist(self.egginfo_dir, self.distinfo_dir) self.write_wheelfile(self.distinfo_dir) self.write_record(self.bdist_dir, self.distinfo_dir) # Make the archive if not testos.path.exists(self.dist_dir): testos.makedirs(self.dist_dir) wheel_name = archive_wheelfile(pseudoinstall_root, archive_root) # Sign the archive if 'WHEEL_TOOL' in testos.environ: subprocess.call([testos.environ['WHEEL_TOOL'], 'sign', wheel_name]) # Add to 'Distribution.dist_files' so that the "upload" command works getattr(self.distribution, 'dist_files', []).append( ('bdist_wheel', get_python_version(), wheel_name)) if not self.keep_temp: if self.dry_run: logger.info('removing %s', self.bdist_dir) else: rmtree(self.bdist_dir)
def save_config_path(*resource): appdata = get_path("CSIDL_LOCAL_APPDATA") path = testos.path.join(appdata, *resource) if not testos.path.isdir(path): testos.makedirs(path) return path
def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, owner=None, group=None, logger=None): """Create a (possibly compressed) tar file from all the files under 'base_dir'. 'compress' must be "gzip" (the default), "bzip2", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus the appropriate compression extension (".gz", or ".bz2"). Returns the output filename. """ tar_compression = {'gzip': 'gz', None: ''} compress_ext = {'gzip': '.gz'} if _BZ2_SUPPORTED: tar_compression['bzip2'] = 'bz2' compress_ext['bzip2'] = '.bz2' # flags for compression program, each element of list will be an argument if compress is not None and compress not in compress_ext: raise ValueError("bad value for 'compress', or compression format not " "supported : {0}".format(compress)) archive_name = base_name + '.tar' + compress_ext.get(compress, '') archive_dir = testos.path.dirname(archive_name) if not testos.path.exists(archive_dir): if logger is not None: logger.info("creating %s", archive_dir) if not dry_run: testos.makedirs(archive_dir) # creating the tarball if logger is not None: logger.info('Creating tar archive') uid = _get_uid(owner) gid = _get_gid(group) def _set_uid_gid(tarinfo): if gid is not None: tarinfo.gid = gid tarinfo.gname = group if uid is not None: tarinfo.uid = uid tarinfo.uname = owner return tarinfo if not dry_run: tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) try: tar.add(base_dir, filter=_set_uid_gid) finally: tar.close() return archive_name