Esempio n. 1
0
def _detectedSourceFile(filename, module_name, result, is_late):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(filename=filename,
                            module_name="_collections_abc",
                            result=result,
                            is_late=is_late)

    source_code = readSourceCodeFromFilename(filename)

    if Utils.python_version >= 300:
        filename = filename.decode("utf-8")

    if module_name == "site":
        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep, os.path.basename(filename), source_code)

    debug("Freezing module '%s' (from '%s').", module_name, filename)

    result.append(
        (module_name, marshal.dumps(compile(source_code, filename, "exec")),
         Utils.basename(filename) == "__init__.py", filename, is_late))

    module_names.add(module_name)
Esempio n. 2
0
def checkRequirements(filename):
    for line in readSourceCodeFromFilename(None, filename).splitlines():
        if line.startswith("# nuitka-skip-unless-"):
            if line[21:33] == "expression: ":
                expression = line[33:]
                with open(os.devnull, "w") as devnull:
                    result = subprocess.call(
                        (
                            os.environ["PYTHON"],
                            "-c",
                            "import sys, os; sys.exit(not bool(%s))" % expression,
                        ),
                        stdout=devnull,
                        stderr=subprocess.STDOUT,
                    )
                if result != 0:
                    return (False, "Expression '%s' evaluated to false" % expression)

            elif line[21:30] == "imports: ":
                imports_needed = line[30:].rstrip().split(",")
                for i in imports_needed:
                    if not hasModule(i):
                        return (
                            False,
                            i
                            + " not installed for this Python version, but test needs it",
                        )
    # default return value
    return (True, "")
Esempio n. 3
0
def _detectedSourceFile(filename, module_name, result, user_provided):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(filename=filename,
                            module_name="_collections_abc",
                            result=result,
                            user_provided=user_provided)

    source_code = readSourceCodeFromFilename(module_name, filename)

    if module_name == "site":
        if source_code.startswith("def ") or source_code.startswith("class "):
            source_code = '\n' + source_code

        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep, Utils.basename(filename), source_code)

    debug("Freezing module '%s' (from '%s').", module_name, filename)

    result.append(
        makeUncompiledPythonModule(
            module_name=module_name,
            bytecode=marshal.dumps(compile(source_code, filename, "exec")),
            is_package=Utils.basename(filename) == "__init__.py",
            filename=filename,
            user_provided=user_provided))

    module_names.add(module_name)
Esempio n. 4
0
def demoteCompiledModuleToBytecode(module):
    """ Demote a compiled module to uncompiled (bytecode).

    """
    full_name = module.getFullName()
    filename = module.getCompileTimeFilename()

    source_code = readSourceCodeFromFilename(full_name, filename)

    source_code = Plugins.onFrozenModuleSourceCode(module_name=full_name,
                                                   is_package=False,
                                                   source_code=source_code)

    bytecode = compile(source_code, filename, "exec")

    bytecode = Plugins.onFrozenModuleBytecode(module_name=full_name,
                                              is_package=False,
                                              bytecode=bytecode)

    uncompiled_module = makeUncompiledPythonModule(
        module_name=module.getFullName(),
        filename=filename,
        bytecode=marshal.dumps(bytecode),
        is_package=module.isCompiledPythonPackage(),
        user_provided=True,
        technical=False)

    replaceImportedModule(old=module, new=uncompiled_module)

    replaceRootModule(old=module, new=uncompiled_module)

    assert module.constraint_collection is not None
    uncompiled_module.setUsedModules(
        module.constraint_collection.getUsedModules())
Esempio n. 5
0
 def getSourceCode(self):
     if self.source_code is not None:
         return self.source_code
     else:
         return readSourceCodeFromFilename(
             module_name=self.getFullName(),
             source_filename=self.getCompileTimeFilename(),
         )
Esempio n. 6
0
def _detectedSourceFile(filename, module_name, result, user_provided,
                        technical):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(filename=filename,
                            module_name="_collections_abc",
                            result=result,
                            user_provided=user_provided,
                            technical=technical)

    source_code = readSourceCodeFromFilename(module_name, filename)

    if module_name == "site":
        if source_code.startswith("def ") or source_code.startswith("class "):
            source_code = '\n' + source_code

        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep, os.path.basename(filename), source_code)

        # Debian stretch site.py
        source_code = source_code.replace(
            "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")

        # Anaconda3 4.1.2 site.py
        source_code = source_code.replace(
            "def main():",
            "def main():return\n\nif 0:\n def _unused():",
        )

    debug("Freezing module '%s' (from '%s').", module_name, filename)

    is_package = os.path.basename(filename) == "__init__.py"
    source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
                                                   is_package=is_package,
                                                   source_code=source_code)

    bytecode = compile(source_code, filename, "exec", dont_inherit=True)

    bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
                                              is_package=is_package,
                                              bytecode=bytecode)

    uncompiled_module = makeUncompiledPythonModule(
        module_name=module_name,
        bytecode=marshal.dumps(bytecode),
        is_package=is_package,
        filename=filename,
        user_provided=user_provided,
        technical=technical)

    ImportCache.addImportedModule(uncompiled_module)

    result.append(uncompiled_module)
    module_names.add(module_name)
Esempio n. 7
0
def recurseTo(module_package, module_filename, module_relpath, module_kind,
              reason):
    from nuitka.tree import Building

    if not ImportCache.isImportedModuleByPath(module_relpath):
        module, source_ref, source_filename = Building.decideModuleTree(
            filename=module_filename,
            package=module_package,
            is_top=False,
            is_main=False,
            is_shlib=module_kind == "shlib")

        # Check if the module name is known. In order to avoid duplicates,
        # learn the new filename, and continue build if its not.
        if not ImportCache.isImportedModuleByName(module.getFullName()):
            debug("Recurse to import '%s' from %s. (%s)", module.getFullName(),
                  module_relpath, reason)

            if module_kind == "py" and source_filename is not None:
                try:
                    Building.createModuleTree(
                        module=module,
                        source_ref=source_ref,
                        source_code=readSourceCodeFromFilename(
                            source_filename),
                        is_main=False)
                except (SyntaxError, IndentationError) as e:
                    if module_filename not in Importing.warned_about:
                        Importing.warned_about.add(module_filename)

                        warning(
                            """\
Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath,
                            module_filename, e.__class__.__name__)

                    return None, False

            ImportCache.addImportedModule(module_relpath, module)

            is_added = True
        else:
            ImportCache.addImportedModule(
                module_relpath,
                ImportCache.getImportedModuleByName(module.getFullName()))

            module = ImportCache.getImportedModuleByName(module.getFullName())

            is_added = False

        assert not module_relpath.endswith("/__init__.py"), module

        return module, is_added
    else:
        return ImportCache.getImportedModuleByPath(module_relpath), False
Esempio n. 8
0
def _detectedSourceFile(filename, module_name, result, is_late):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(
            filename    = filename,
            module_name = "_collections_abc",
            result      = result,
            is_late     = is_late
        )

    source_code = readSourceCodeFromFilename(module_name, filename)

    if Utils.python_version >= 300:
        filename = filename.decode("utf-8")

    if module_name == "site":
        if source_code.startswith("def ") or source_code.startswith("class "):
            source_code = '\n' + source_code

        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep,
            Utils.basename(filename),
            source_code
        )

    debug(
        "Freezing module '%s' (from '%s').",
        module_name,
        filename
    )

    result.append(
        (
            module_name,
            marshal.dumps(
                compile(source_code, filename, "exec")
            ),
            Utils.basename(filename) == "__init__.py",
            filename,
            is_late
        )
    )

    module_names.add(module_name)
Esempio n. 9
0
def demoteCompiledModuleToBytecode(module):
    """ Demote a compiled module to uncompiled (bytecode).

    """
    full_name = module.getFullName()
    filename = module.getCompileTimeFilename()

    source_code = readSourceCodeFromFilename(full_name, filename)

    source_code = Plugins.onFrozenModuleSourceCode(
        module_name = full_name,
        is_package  = False,
        source_code = source_code
    )

    bytecode = compile(source_code, filename, "exec")

    bytecode = Plugins.onFrozenModuleBytecode(
        module_name = full_name,
        is_package  = False,
        bytecode    = bytecode
    )

    uncompiled_module = makeUncompiledPythonModule(
        module_name   = module.getFullName(),
        filename      = filename,
        bytecode      = marshal.dumps(bytecode),
        is_package    = module.isCompiledPythonPackage(),
        user_provided = True,
        technical     = False
    )

    replaceImportedModule(
        old = module,
        new = uncompiled_module
    )

    replaceRootModule(
        old = module,
        new = uncompiled_module
    )

    assert module.constraint_collection is not None
    uncompiled_module.setUsedModules(module.constraint_collection.getUsedModules())
Esempio n. 10
0
def _detectedSourceFile(filename, module_name, result, user_provided):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(
            filename      = filename,
            module_name   = "_collections_abc",
            result        = result,
            user_provided = user_provided
        )

    source_code = readSourceCodeFromFilename(module_name, filename)

    if module_name == "site":
        if source_code.startswith("def ") or source_code.startswith("class "):
            source_code = '\n' + source_code

        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep,
            Utils.basename(filename),
            source_code
        )

    debug(
        "Freezing module '%s' (from '%s').",
        module_name,
        filename
    )

    result.append(
        makeUncompiledPythonModule(
            module_name   = module_name,
            bytecode      = marshal.dumps(
                compile(source_code, filename, "exec")
            ),
            is_package    = Utils.basename(filename) == "__init__.py",
            filename      = filename,
            user_provided = user_provided
        )
    )

    module_names.add(module_name)
Esempio n. 11
0
    def createFakeModuleDependency(module):
        full_name = module.getFullName()

        if full_name != "multiprocessing":
            return

        # First, build the module node and then read again from the
        # source code.
        root_module = getRootTopModule()

        module_name = ModuleName("__parents_main__")

        source_code = readSourceCodeFromFilename(module_name,
                                                 root_module.getFilename())

        # For the call stack, this may look bad or different to what
        # CPython does. Using the "__import__" built-in to not spoil
        # or use the module namespace. The forking module was split up
        # into multiple modules in Python 3.4.
        if python_version >= 0x340:
            source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
# Not needed, and can crash from minor __file__ differences, depending on invocation
__import__("multiprocessing.spawn").spawn._fixup_main_from_path = lambda mod_name : None
__import__("multiprocessing.spawn").spawn.freeze_support()"""
        else:
            source_code += """
__import__("sys").modules["__main__"] = __import__("sys").modules[__name__]
__import__("multiprocessing.forking").forking.freeze_support()"""

        yield (
            module_name,
            source_code,
            root_module.getCompileTimeFilename(),
            "Autoenable multiprocessing freeze support",
        )
Esempio n. 12
0
def recurseTo(module_package, module_filename, module_relpath, module_kind,
             reason):
    from nuitka.tree import Building
    from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule

    if not ImportCache.isImportedModuleByPath(module_relpath):
        module, source_ref, source_filename = Building.decideModuleTree(
            filename = module_filename,
            package  = module_package,
            is_top   = False,
            is_main  = False,
            is_shlib = module_kind == "shlib"
        )

        # Check if the module name is known. In order to avoid duplicates,
        # learn the new filename, and continue build if its not.
        if not ImportCache.isImportedModuleByName(module.getFullName()):
            debug(
                "Recurse to import '%s' from %s. (%s)",
                module.getFullName(),
                module_relpath,
                reason
            )

            if module_kind == "py" and source_filename is not None:
                try:
                    Building.createModuleTree(
                        module      = module,
                        source_ref  = source_ref,
                        source_code = readSourceCodeFromFilename(
                            module_name     = module.getFullName(),
                            source_filename = source_filename
                        ),
                        is_main     = False
                    )
                except (SyntaxError, IndentationError) as e:
                    if module_filename not in Importing.warned_about:
                        Importing.warned_about.add(module_filename)

                        warning(
                            """\
Cannot recurse to import module '%s' (%s) because of '%s'""",
                            module_relpath,
                            module_filename,
                            e.__class__.__name__
                        )

                    return None, False
                except Building.CodeTooComplexCode:
                    if module_filename not in Importing.warned_about:
                        Importing.warned_about.add(module_filename)

                        warning(
                            """\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
                            module_relpath,
                            module_filename,
                        )


                        if Options.isStandaloneMode():
                            module = makeUncompiledPythonModule(
                                module_name   = module.getFullName(),
                                filename      = module_filename,
                                bytecode      = marshal.dumps(
                                    compile(
                                        readSourceCodeFromFilename(module.getFullName(), module_filename),
                                        module_filename,
                                        "exec"
                                    )
                                ),
                                is_package    = module.isCompiledPythonPackage(),
                                user_provided = True,
                                technical     = False
                            )

                            ModuleRegistry.addUncompiledModule(module)

                    return None, False

            ImportCache.addImportedModule(module)

            is_added = True
        else:
            module = ImportCache.getImportedModuleByName(
                module.getFullName()
            )

            is_added = False

        assert not module_relpath.endswith("/__init__.py"), module

        return module, is_added
    else:
        return ImportCache.getImportedModuleByPath(module_relpath), False
Esempio n. 13
0
def _recurseTo(module_package, module_filename, module_relpath, module_kind, reason):
    from nuitka.tree import Building
    from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule

    module, source_ref, source_filename = Building.decideModuleTree(
        filename=module_filename,
        package=module_package,
        is_top=False,
        is_main=False,
        is_shlib=module_kind == "shlib",
    )

    logRecursion(
        "Recurse to import '%s' from '%s'. (%s)",
        module.getFullName(),
        module_relpath,
        reason,
    )

    if module_kind == "py" and source_filename is not None:
        try:
            source_code = readSourceCodeFromFilename(
                module_name=module.getFullName(), source_filename=source_filename
            )

            Building.createModuleTree(
                module=module,
                source_ref=source_ref,
                source_code=source_code,
                is_main=False,
            )
        except (SyntaxError, IndentationError) as e:
            if module_filename not in Importing.warned_about:
                Importing.warned_about.add(module_filename)

                warning(
                    """\
Cannot recurse to import module '%s' (%s) because of '%s'""",
                    module_relpath,
                    module_filename,
                    e.__class__.__name__,
                )

            return None, False
        except Building.CodeTooComplexCode:
            if module_filename not in Importing.warned_about:
                Importing.warned_about.add(module_filename)

                warning(
                    """\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
                    module_relpath,
                    module_filename,
                )

                if Options.isStandaloneMode():
                    module = makeUncompiledPythonModule(
                        module_name=module.getFullName(),
                        filename=module_filename,
                        bytecode=marshal.dumps(
                            compile(
                                source_code, module_filename, "exec", dont_inherit=True
                            )
                        ),
                        is_package=module.isCompiledPythonPackage(),
                        user_provided=True,
                        technical=False,
                    )

                    ModuleRegistry.addUncompiledModule(module)

            return None, False

    ImportCache.addImportedModule(module)

    return module, True
Esempio n. 14
0
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
               reason):
    from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule
    from nuitka.tree import Building

    module, source_ref, source_filename = Building.decideModuleTree(
        filename=module_filename,
        package=module_package,
        is_top=False,
        is_main=False,
        is_shlib=module_kind == "shlib",
    )

    if Options.isShowInclusion():
        recursion_logger.info("Recurse to import '%s' from '%s'. (%s)" %
                              (module.getFullName(), module_relpath, reason))

    if module_kind == "py" and source_filename is not None:
        try:
            source_code = readSourceCodeFromFilename(
                module_name=module.getFullName(),
                source_filename=source_filename)

            Building.createModuleTree(
                module=module,
                source_ref=source_ref,
                source_code=source_code,
                is_main=False,
            )
        except (SyntaxError, IndentationError) as e:
            if module_filename not in Importing.warned_about:
                Importing.warned_about.add(module_filename)

                warning(
                    """\
Cannot recurse to import module '%s' (%s) because of '%s'""",
                    module_relpath,
                    module_filename,
                    e.__class__.__name__,
                )

            return None, False
        except Building.CodeTooComplexCode:
            if module_filename not in Importing.warned_about:
                Importing.warned_about.add(module_filename)

                warning(
                    """\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
                    module_relpath,
                    module_filename,
                )

                if Options.isStandaloneMode():
                    module = makeUncompiledPythonModule(
                        module_name=module.getFullName(),
                        filename=module_filename,
                        bytecode=marshal.dumps(
                            compile(source_code,
                                    module_filename,
                                    "exec",
                                    dont_inherit=True)),
                        is_package=module.isCompiledPythonPackage(),
                        user_provided=True,
                        technical=False,
                    )

                    ModuleRegistry.addUncompiledModule(module)

            return None, False

    ImportCache.addImportedModule(module)

    return module, True
Esempio n. 15
0
def recurseTo(module_package, module_filename, module_relpath, module_kind,
             reason):
    from nuitka.tree import Building

    if not ImportCache.isImportedModuleByPath(module_relpath):
        module, source_ref, source_filename = Building.decideModuleTree(
            filename = module_filename,
            package  = module_package,
            is_top   = False,
            is_main  = False,
            is_shlib = module_kind == "shlib"
        )

        # Check if the module name is known. In order to avoid duplicates,
        # learn the new filename, and continue build if its not.
        if not ImportCache.isImportedModuleByName(module.getFullName()):
            debug(
                "Recurse to import '%s' from %s. (%s)",
                module.getFullName(),
                module_relpath,
                reason
            )

            if module_kind == "py" and source_filename is not None:
                try:
                    Building.createModuleTree(
                        module      = module,
                        source_ref  = source_ref,
                        source_code = readSourceCodeFromFilename(
                            module_name     = module.getFullName(),
                            source_filename = source_filename
                        ),
                        is_main     = False
                    )
                except (SyntaxError, IndentationError) as e:
                    if module_filename not in Importing.warned_about:
                        Importing.warned_about.add(module_filename)

                        warning(
                            """\
Cannot recurse to import module '%s' (%s) because of '%s'""",
                            module_relpath,
                            module_filename,
                            e.__class__.__name__
                        )

                    return None, False

            ImportCache.addImportedModule(
                module_relpath,
                module
            )

            is_added = True
        else:
            ImportCache.addImportedModule(
                module_relpath,
                ImportCache.getImportedModuleByName(module.getFullName())
            )

            module = ImportCache.getImportedModuleByName(
                module.getFullName()
            )

            is_added = False

        assert not module_relpath.endswith("/__init__.py"), module

        return module, is_added
    else:
        return ImportCache.getImportedModuleByPath(module_relpath), False
Esempio n. 16
0
def _recurseTo(module_package, module_filename, module_relpath, module_kind,
               reason):
    from nuitka.tree import Building
    from nuitka.nodes.ModuleNodes import makeUncompiledPythonModule

    module, source_ref, source_filename = Building.decideModuleTree(
        filename=module_filename,
        package=module_package,
        is_top=False,
        is_main=False,
        is_shlib=module_kind == "shlib")

    # Check if the module name is known. In order to avoid duplicates,
    # learn the new filename, and continue build if its not.
    if not ImportCache.isImportedModuleByName(module.getFullName()):
        logRecursion("Recurse to import '%s' from '%s'. (%s)",
                     module.getFullName(), module_relpath, reason)

        if module_kind == "py" and source_filename is not None:
            try:
                source_code = readSourceCodeFromFilename(
                    module_name=module.getFullName(),
                    source_filename=source_filename)

                Building.createModuleTree(module=module,
                                          source_ref=source_ref,
                                          source_code=source_code,
                                          is_main=False)
            except (SyntaxError, IndentationError) as e:
                if module_filename not in Importing.warned_about:
                    Importing.warned_about.add(module_filename)

                    warning(
                        """\
Cannot recurse to import module '%s' (%s) because of '%s'""", module_relpath,
                        module_filename, e.__class__.__name__)

                return None, False
            except Building.CodeTooComplexCode:
                if module_filename not in Importing.warned_about:
                    Importing.warned_about.add(module_filename)

                    warning(
                        """\
Cannot recurse to import module '%s' (%s) because code is too complex.""",
                        module_relpath,
                        module_filename,
                    )

                    if Options.isStandaloneMode():
                        module = makeUncompiledPythonModule(
                            module_name=module.getFullName(),
                            filename=module_filename,
                            bytecode=marshal.dumps(
                                compile(source_code,
                                        module_filename,
                                        "exec",
                                        dont_inherit=True)),
                            is_package=module.isCompiledPythonPackage(),
                            user_provided=True,
                            technical=False)

                        ModuleRegistry.addUncompiledModule(module)

                return None, False

        ImportCache.addImportedModule(module)

        is_added = True
    else:
        module = ImportCache.getImportedModuleByName(module.getFullName())

        is_added = False

    return module, is_added
Esempio n. 17
0
def _detectedSourceFile(filename, module_name, result, user_provided,
                        technical):
    if module_name in module_names:
        return

    if module_name == "collections.abc":
        _detectedSourceFile(
            filename=filename,
            module_name=ModuleName("_collections_abc"),
            result=result,
            user_provided=user_provided,
            technical=technical,
        )

    source_code = readSourceCodeFromFilename(module_name, filename)

    if module_name == "site":
        if source_code.startswith("def ") or source_code.startswith("class "):
            source_code = "\n" + source_code

        source_code = """\
__file__ = (__nuitka_binary_dir + '%s%s') if '__nuitka_binary_dir' in dict(__builtins__ ) else '<frozen>';%s""" % (
            os.path.sep,
            os.path.basename(filename),
            source_code,
        )

        # Debian stretch site.py
        source_code = source_code.replace(
            "PREFIXES = [sys.prefix, sys.exec_prefix]", "PREFIXES = []")

    if Options.isShowInclusion():
        inclusion_logger.info("Freezing module '%s' (from '%s')." %
                              (module_name, filename))

    is_package = os.path.basename(filename) == "__init__.py"

    # Plugins can modify source code:
    source_code = Plugins.onFrozenModuleSourceCode(module_name=module_name,
                                                   is_package=is_package,
                                                   source_code=source_code)

    bytecode = compileSourceToBytecode(
        source_code=source_code,
        filename=module_name.replace(".", os.path.sep) + ".py",
    )

    # Plugins can modify bytecode code:
    bytecode = Plugins.onFrozenModuleBytecode(module_name=module_name,
                                              is_package=is_package,
                                              bytecode=bytecode)

    uncompiled_module = makeUncompiledPythonModule(
        module_name=module_name,
        bytecode=marshal.dumps(bytecode),
        is_package=is_package,
        filename=filename,
        user_provided=user_provided,
        technical=technical,
    )

    ImportCache.addImportedModule(uncompiled_module)

    result.append(uncompiled_module)
    module_names.add(module_name)