Example #1
0
def test_str0():
    @signature(types.unicode0(), returns=types.str0())
    def f(u):
        return 'str'

    assert getsig(f) == [
        model.SomeUnicodeString(no_nul=True),
        model.SomeString(no_nul=True)
    ]
Example #2
0
def test_str0():
    @signature(types.unicode0(), returns=types.str0())
    def f(u):
        return 'str'
    assert getsig(f) == [model.SomeUnicodeString(no_nul=True),
                         model.SomeString(no_nul=True)]
Example #3
0
                    _fix_up_module(d, name, pathname, cpathname)
            """)
        #
    code_w.exec_code(space, w_dict, w_dict)


def rightmost_sep(filename):
    "Like filename.rfind('/'), but also search for \\."
    index = filename.rfind(os.sep)
    if os.altsep is not None:
        index2 = filename.rfind(os.altsep)
        index = max(index, index2)
    return index


@signature(types.str0(), returns=types.str0())
def make_compiled_pathname(pathname):
    "Given the path to a .py file, return the path to its .pyc file."
    # foo.py -> __pycache__/foo.<tag>.pyc

    lastpos = rightmost_sep(pathname) + 1
    assert lastpos >= 0  # zero when slash, takes the full name
    fname = pathname[lastpos:]
    if lastpos > 0:
        # Windows: re-use the last separator character (/ or \\) when
        # appending the __pycache__ path.
        lastsep = pathname[lastpos - 1]
    else:
        lastsep = os.sep
    ext = fname
    for i in range(len(fname)):
Example #4
0
class W_ZipImporter(W_Root):
    def __init__(self, space, name, filename, zip_file, prefix):
        self.space = space
        self.name = name
        self.filename = filename
        self.zip_file = zip_file
        self.prefix = prefix

    def getprefix(self, space):
        if ZIPSEP == os.path.sep:
            return space.newfilename(self.prefix)
        return space.newfilename(self.prefix.replace(ZIPSEP, os.path.sep))

    def _find_relative_path(self, filename):
        if filename.startswith(self.filename):
            filename = filename[len(self.filename):]
        if filename.startswith(os.path.sep) or filename.startswith(ZIPSEP):
            filename = filename[1:]
        if ZIPSEP != os.path.sep:
            filename = filename.replace(os.path.sep, ZIPSEP)
        return filename

    @signature(types.self(), types.str0(), returns=types.str0())
    def corr_zname(self, fname):
        if ZIPSEP != os.path.sep:
            return fname.replace(ZIPSEP, os.path.sep)
        else:
            return fname

    @enforceargs(filename=s_Str0, typecheck=False)
    def import_py_file(self, space, modname, filename, buf, pkgpath):
        w_mod = Module(space, space.newtext(modname))
        real_name = self.filename + os.path.sep + self.corr_zname(filename)
        space.setattr(w_mod, space.newtext('__loader__'), self)
        importing._prepare_module(space, w_mod, real_name, pkgpath)
        co_filename = self.make_co_filename(filename)
        code_w = importing.parse_source_module(space, co_filename, buf)
        importing.exec_code_module(space, w_mod, code_w, co_filename, None)
        return w_mod

    def _parse_mtime(self, space, filename):
        try:
            info = self.zip_file.NameToInfo[filename]
            t = info.date_time
        except KeyError:
            return 0
        else:
            w_mktime = space.getattr(space.getbuiltinmodule('time'),
                                     space.newtext('mktime'))
            # XXX this is incredible fishing around module limitations
            #     in order to compare timestamps of .py and .pyc files
            # we need time.mktime support on rpython level
            w = space.newint
            all = [
                w(t[0]),
                w(t[1]),
                w(t[2]),
                w(t[3]),
                w(t[4]),
                w(t[5]),
                w(0),
                w(1),
                w(-1)
            ]
            mtime = int(
                space.float_w(
                    space.call_function(w_mktime, space.newtuple(all))))
            return mtime

    def check_newer_pyfile(self, space, filename, timestamp):
        # check if the timestamp stored in the .pyc is matching
        # the actual timestamp of the .py file, if any
        mtime = self._parse_mtime(space, filename)
        if mtime == 0:
            return False
        # Lenient date/time comparison function. The precision of the mtime
        # in the archive is lower than the mtime stored in a .pyc: we
        # must allow a difference of at most one second.
        d = mtime - timestamp
        if d < 0:
            d = -d
        return d > 1  # more than one second => different

    def can_use_pyc(self, space, filename, magic, buf):
        if magic != importing.get_pyc_magic(space):
            return False
        bitfield = importing._get_long(buf[4:8])
        if bitfield != 0:
            # Hash-based pyc. CPython (and we) currently refuse to handle
            # checked hash-based pycs in zips. We could validate hash-based
            # pycs against the source, but it seems likely that most people
            # putting hash-based pycs in a zipfile will use unchecked ones.
            w_imp = space.getbuiltinmodule("_imp")
            w_mode = space.getattr(w_imp,
                                   space.newtext("check_hash_based_pycs"))
            mode = space.text_w(w_mode)
            if mode == "always":
                return False
            if mode == "default" and bitfield & 0b10:
                return False
            return True
        timestamp = importing._get_long(buf[8:12])
        if self.check_newer_pyfile(space, filename[:-1], timestamp):
            return False
        return True

    @enforceargs(filename=s_Str0, typecheck=False)
    def import_pyc_file(self, space, modname, filename, buf, pkgpath):
        # a field are four bytes
        # | magic | 0b00 | timestamp | size   # traditional timestamp based pyc
        # | magic | 0b01 | hash1     | hash2  # unchecked
        # | magic | 0b11 | hash1     | hash2  # checked

        if len(buf) < 16:
            raise oefmt(get_error(space), "bad pyc data")
        magic = importing._get_long(buf[:4])
        if not self.can_use_pyc(space, filename, magic, buf):
            return None
        buf = buf[16:]  # XXX ugly copy, should use sequential read instead
        w_mod = Module(space, space.newtext(modname))
        real_name = self.filename + os.path.sep + self.corr_zname(filename)
        space.setattr(w_mod, space.newtext('__loader__'), self)
        importing._prepare_module(space, w_mod, real_name, pkgpath)
        result = importing.load_compiled_module(space, space.newtext(modname),
                                                w_mod, real_name, magic, buf)
        return result

    def have_modulefile(self, space, filename):
        if ZIPSEP != os.path.sep:
            filename = filename.replace(os.path.sep, ZIPSEP)
        try:
            self.zip_file.NameToInfo[filename]
            return True
        except KeyError:
            return False

    @unwrap_spec(fullname='text')
    def find_module(self, space, fullname, w_path=None):
        filename = self.make_filename(fullname)
        for _, _, ext in ENUMERATE_EXTS:
            if self.have_modulefile(space, filename + ext):
                return self

    def make_filename(self, fullname):
        startpos = fullname.rfind('.') + 1  # 0 when not found
        assert startpos >= 0
        subname = fullname[startpos:]
        if ZIPSEP == os.path.sep:
            return self.prefix + subname.replace('.', '/')
        else:
            return (self.prefix.replace(os.path.sep, ZIPSEP) +
                    subname.replace('.', '/'))

    def make_co_filename(self, filename):
        """
        Return the filename to be used for compiling the module, i.e. what
        gets in code_object.co_filename. Something like
        'myfile.zip/mymodule.py'
        """
        if ZIPSEP != os.path.sep:
            filename = filename.replace(ZIPSEP, os.path.sep)
        return self.filename + os.path.sep + filename

    def load_module(self, space, w_fullname):
        fullname = space.text_w(w_fullname)
        filename = self.make_filename(fullname)
        for compiled, is_package, ext in ENUMERATE_EXTS:
            if '\x00' in filename:
                # Special case to make the annotator happy:
                # filenames inside ZIPs shouldn't contain NULs so no module can
                # possibly be found in this case
                break
            filename = assert_str0(filename)
            fname = filename + ext
            try:
                buf = self.zip_file.read(fname)
            except (KeyError, OSError, BadZipfile):
                pass
            except RZlibError as e:
                # in this case, CPython raises the direct exception coming
                # from the zlib module: let's do the same
                raise zlib_error(space, e.msg)
            else:
                if is_package:
                    pkgpath = (self.filename + os.path.sep +
                               self.corr_zname(filename))
                else:
                    pkgpath = None
                try:
                    if compiled:
                        w_result = self.import_pyc_file(
                            space, fullname, fname, buf, pkgpath)
                        if w_result is None:
                            continue
                    else:
                        w_result = self.import_py_file(space, fullname, fname,
                                                       buf, pkgpath)
                    if space.sys.get_flag('verbose') >= 1:
                        w_stderr = space.sys.get('stderr')
                        message = "import %s # loaded from Zip %s%s%s\n" % (
                            fullname, self.filename, os.path.sep, fname)
                        space.call_method(w_stderr, "write",
                                          space.newtext(message))
                    return w_result
                except:
                    w_mods = space.sys.get('modules')
                    space.call_method(w_mods, 'pop', w_fullname, space.w_None)
                    raise
        raise oefmt(get_error(space), "can't find module %R", w_fullname)

    @unwrap_spec(filename='text')
    def get_data(self, space, filename):
        filename = self._find_relative_path(filename)
        try:
            data = self.zip_file.read(filename)
            return space.newbytes(data)
        except (KeyError, OSError, BadZipfile):
            raise oefmt(space.w_IOError, "Error reading file")
        except RZlibError as e:
            # in this case, CPython raises the direct exception coming
            # from the zlib module: let's do the same
            raise zlib_error(space, e.msg)

    def get_code(self, space, w_fullname):
        fullname = space.text_w(w_fullname)
        filename = self.make_filename(fullname)
        for compiled, _, ext in ENUMERATE_EXTS:
            if '\x00' in filename:
                # Special case to make the annotator happy:
                # filenames inside ZIPs shouldn't contain NULs so no module can
                # possibly be found in this case
                break
            filename = assert_str0(filename)
            if self.have_modulefile(space, filename + ext):
                w_source = self.get_data(space, filename + ext)
                source = space.bytes_w(w_source)
                if compiled:
                    if len(source) < 16:
                        raise oefmt(get_error(space), "bad pyc data")
                    magic = importing._get_long(source[:4])
                    if not self.can_use_pyc(space, filename + ext, magic,
                                            source):
                        continue
                    # zipimport ignores the size field
                    w_code = importing.read_compiled_module(
                        space, filename + ext, source[16:])
                else:
                    co_filename = self.make_co_filename(filename + ext)
                    w_code = importing.parse_source_module(
                        space, co_filename, source)
                return w_code
        raise oefmt(get_error(space),
                    "Cannot find source or code for %R in %R", w_fullname,
                    space.newfilename(self.name))

    @unwrap_spec(fullname='text')
    def get_source(self, space, fullname):
        filename = self.make_filename(fullname)
        found = False
        for compiled, _, ext in ENUMERATE_EXTS:
            if '\x00' in filename:
                # Special case to make the annotator happy:
                # filenames inside ZIPs shouldn't contain NULs so no module can
                # possibly be found in this case
                break
            filename = assert_str0(filename)
            fname = filename + ext
            if self.have_modulefile(space, fname):
                if not compiled:
                    w_data = self.get_data(space, fname)
                    # XXX CPython does not handle the coding cookie either.
                    return space.call_method(w_data, "decode",
                                             space.newtext("utf-8"))
                else:
                    found = True
        if found:
            # We have the module, but no source.
            return space.w_None
        raise oefmt(get_error(space), "Cannot find source for %R in %R",
                    space.newfilename(filename), space.newfilename(self.name))

    def get_filename(self, space, w_fullname):
        fullname = space.text_w(w_fullname)
        filename = self.make_filename(fullname)
        for _, is_package, ext in ENUMERATE_EXTS:
            if '\x00' in filename:
                # Special case to make the annotator happy:
                # filenames inside ZIPs shouldn't contain NULs so no module can
                # possibly be found in this case
                break
            filename = assert_str0(filename)
            if self.have_modulefile(space, filename + ext):
                return space.newfilename(self.filename + os.path.sep +
                                         self.corr_zname(filename + ext))
        raise oefmt(get_error(space), "Cannot find module %R in %R",
                    space.newfilename(filename), space.newfilename(self.name))

    def is_package(self, space, w_fullname):
        fullname = space.text_w(w_fullname)
        filename = self.make_filename(fullname)
        for _, is_package, ext in ENUMERATE_EXTS:
            if '\x00' in filename:
                # Special case to make the annotator happy:
                # filenames inside ZIPs shouldn't contain NULs so no module can
                # possibly be found in this case
                break
            filename = assert_str0(filename)
            if self.have_modulefile(space, filename + ext):
                return space.newbool(is_package)
        raise oefmt(get_error(space), "Cannot find module %R in %R",
                    space.newfilename(filename), space.newfilename(self.name))

    def getarchive(self, space):
        space = self.space
        return space.newfilename(self.filename)

    def _find_loader(self, space, fullname):
        if '\x00' in fullname:
            # Special case to make the annotator happy:
            # filenames inside ZIPs shouldn't contain NULs so no module can
            # possibly be found in this case
            return False, None
        fullname = assert_str0(fullname)
        filename = self.make_filename(fullname)
        filename = assert_str0(filename)
        for _, _, ext in ENUMERATE_EXTS:
            if self.have_modulefile(space, filename + ext):
                return True, None
        # See if this is a directory (part of a namespace pkg)
        dirpath = self.prefix + fullname.split(".")[-1]
        if self.have_modulefile(space, dirpath + ZIPSEP):
            return True, self.filename + os.path.sep + self.corr_zname(dirpath)
        return False, None

    @unwrap_spec(fullname='text')
    def find_loader(self, space, fullname, w_path=None):
        found, ns_portion = self._find_loader(space, fullname)
        if not found:
            result = [space.w_None, space.newlist([])]
        elif not ns_portion:
            result = [self, space.newlist([])]
        else:
            result = [
                space.w_None,
                space.newlist([space.newfilename(ns_portion)])
            ]
        return space.newtuple(result)
Example #5
0
            w_pathname = get_sourcefile(space, pathname)
        else:
            w_pathname = space.wrap(code_w.co_filename)
        space.setitem(w_dict, space.wrap("__file__"), w_pathname)
        space.setitem(w_dict, space.wrap("__cached__"), space.wrap(cpathname))
    code_w.exec_code(space, w_dict, w_dict)

def rightmost_sep(filename):
    "Like filename.rfind('/'), but also search for \\."
    index = filename.rfind(os.sep)
    if os.altsep is not None:
        index2 = filename.rfind(os.altsep)
        index = max(index, index2)
    return index

@signature(types.str0(), returns=types.str0())
def make_compiled_pathname(pathname):
    "Given the path to a .py file, return the path to its .pyc file."
    # foo.py -> __pycache__/foo.<tag>.pyc

    lastpos = rightmost_sep(pathname) + 1
    assert lastpos >= 0  # zero when slash, takes the full name
    fname = pathname[lastpos:]
    if lastpos > 0:
        # Windows: re-use the last separator character (/ or \\) when
        # appending the __pycache__ path.
        lastsep = pathname[lastpos-1]
    else:
        lastsep = os.sep
    ext = fname
    for i in range(len(fname)):