Exemple #1
0
    def task_source_jvm(self):
        from pypy.translator.jvm.genjvm import GenJvm
        from pypy.translator.jvm.node import EntryPoint

        entry_point_graph = self.translator.graphs[0]
        is_func = not self.standalone
        entry_point = EntryPoint(entry_point_graph, is_func, is_func)
        self.gen = GenJvm(udir, self.translator, entry_point)
        self.jvmsource = self.gen.generate_source()
        self.log.info("Wrote JVM code")
Exemple #2
0
 def task_source_js(self):
     from pypy.translator.js.js import JS
     self.gen = JS(self.translator,
                   functions=[self.entry_point],
                   stackless=self.config.translation.stackless)
     filename = self.gen.write_source()
     self.log.info("Wrote %s" % (filename, ))
Exemple #3
0
    def task_source_jvm(self):
        from pypy.translator.jvm.genjvm import GenJvm
        from pypy.translator.jvm.node import EntryPoint

        entry_point_graph = self.translator.graphs[0]
        is_func = not self.standalone
        entry_point = EntryPoint(entry_point_graph, is_func, is_func)
        self.gen = GenJvm(udir, self.translator, entry_point)
        self.jvmsource = self.gen.generate_source()
        self.log.info("Wrote JVM code")
Exemple #4
0
    def task_source_cli(self):
        from pypy.translator.cli.gencli import GenCli
        from pypy.translator.cli.entrypoint import get_entrypoint

        if self.entry_point is not None: # executable mode
            entry_point_graph = self.translator.graphs[0]
            entry_point = get_entrypoint(entry_point_graph)
        else:
            # library mode
            assert self.libdef is not None
            bk = self.translator.annotator.bookkeeper
            entry_point = self.libdef.get_entrypoint(bk)

        self.gen = GenCli(udir, self.translator, entry_point, config=self.config)
        filename = self.gen.generate_source()
        self.log.info("Wrote %s" % (filename,))
Exemple #5
0
    def task_source_cli(self):
        from pypy.translator.cli.gencli import GenCli
        from pypy.translator.cli.entrypoint import get_entrypoint

        if self.entry_point is not None: # executable mode
            entry_point_graph = self.translator.graphs[0]
            entry_point = get_entrypoint(entry_point_graph)
        else:
            # library mode
            assert self.libdef is not None
            bk = self.translator.annotator.bookkeeper
            entry_point = self.libdef.get_entrypoint(bk)

        self.gen = GenCli(udir, self.translator, entry_point, config=self.config)
        filename = self.gen.generate_source()
        self.log.info("Wrote %s" % (filename,))
Exemple #6
0
class TranslationDriver(SimpleTaskEngine):

    def __init__(self, setopts=None, default_goal=None,
                 disable=[],
                 exe_name=None, extmod_name=None,
                 config=None, overrides=None):
        self.timer = Timer()
        SimpleTaskEngine.__init__(self)

        self.log = log

        if config is None:
            from pypy.config.pypyoption import get_pypy_config
            config = get_pypy_config(DEFAULTS, translating=True)
        self.config = config
        if overrides is not None:
            self.config.override(overrides)

        if setopts is not None:
            self.config.set(**setopts)
        
        self.exe_name = exe_name
        self.extmod_name = extmod_name

        self.done = {}

        self.disable(disable)

        if default_goal:
            default_goal, = self.backend_select_goals([default_goal])
            if default_goal in self._maybe_skip():
                default_goal = None
        
        self.default_goal = default_goal
        self.extra_goals = []
        self.exposed = []

        # expose tasks
        def expose_task(task, backend_goal=None):
            if backend_goal is None:
                backend_goal = task
            def proc():
                return self.proceed(backend_goal)
            self.exposed.append(task)
            setattr(self, task, proc)

        backend, ts = self.get_backend_and_type_system()
        for task in self.tasks:
            explicit_task = task
            parts = task.split('_')
            if len(parts) == 1:
                if task in ('annotate'):
                    expose_task(task)
            else:
                task, postfix = parts
                if task in ('rtype', 'backendopt', 'llinterpret',
                            'prehannotatebackendopt', 'hintannotate',
                            'timeshift'):
                    if ts:
                        if ts == postfix:
                            expose_task(task, explicit_task)
                    else:
                        expose_task(explicit_task)
                elif task in ('source', 'compile', 'run'):
                    if backend:
                        if backend == postfix:
                            expose_task(task, explicit_task)
                    elif ts:
                        if ts == backend_to_typesystem(postfix):
                            expose_task(explicit_task)
                    else:
                        expose_task(explicit_task)

    def set_extra_goals(self, goals):
        self.extra_goals = goals

    def get_info(self): # XXX more?
        d = {'backend': self.config.translation.backend}
        return d

    def get_backend_and_type_system(self):
        type_system = self.config.translation.type_system
        backend = self.config.translation.backend
        return backend, type_system

    def backend_select_goals(self, goals):
        backend, ts = self.get_backend_and_type_system()
        postfixes = [''] + ['_'+p for p in (backend, ts) if p]
        l = []
        for goal in goals:
            for postfix in postfixes:
                cand = "%s%s" % (goal, postfix)
                if cand in self.tasks:
                    new_goal = cand
                    break
            else:
                raise Exception, "cannot infer complete goal from: %r" % goal 
            l.append(new_goal)
        return l

    def disable(self, to_disable):
        self._disabled = to_disable

    def _maybe_skip(self):
        maybe_skip = []
        if self._disabled:
             for goal in  self.backend_select_goals(self._disabled):
                 maybe_skip.extend(self._depending_on_closure(goal))
        return dict.fromkeys(maybe_skip).keys()


    def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
        standalone = inputtypes is None
        self.standalone = standalone

        if standalone:
            inputtypes = [s_list_of_strings]
        self.inputtypes = inputtypes

        if policy is None:
            policy = annpolicy.AnnotatorPolicy()
        if standalone:
            policy.allow_someobjects = False
        self.policy = policy

        self.extra = extra

        if empty_translator:
            translator = empty_translator
        else:
            translator = TranslationContext(config=self.config)

        self.entry_point = entry_point
        self.translator = translator
        self.libdef = None

        self.translator.driver_instrument_result = self.instrument_result

    def setup_library(self, libdef, policy=None, extra={}, empty_translator=None):
        self.setup(None, None, policy, extra, empty_translator)
        self.libdef = libdef

    def instrument_result(self, args):
        backend, ts = self.get_backend_and_type_system()
        if backend != 'c' or sys.platform == 'win32':
            raise Exception("instrumentation requires the c backend"
                            " and unix for now")
        from pypy.tool.udir import udir
        
        datafile = udir.join('_instrument_counters')
        makeProfInstrument = lambda compiler: ProfInstrument(datafile, compiler)

        pid = os.fork()
        if pid == 0:
            # child compiling and running with instrumentation
            self.config.translation.instrument = True
            self.config.translation.instrumentctl = (makeProfInstrument,
                                                     args)
            raise Instrument
        else:
            pid, status = os.waitpid(pid, 0)
            if os.WIFEXITED(status):
                status = os.WEXITSTATUS(status)
                if status != 0:
                    raise Exception, "instrumentation child failed: %d" % status
            else:
                raise Exception, "instrumentation child aborted"
            import array, struct
            n = datafile.size()//struct.calcsize('L')
            datafile = datafile.open('rb')
            counters = array.array('L')
            counters.fromfile(datafile, n)
            datafile.close()
            return counters

    def info(self, msg):
        log.info(msg)

    def _profile(self, goal, func):
        from cProfile import Profile
        from pypy.tool.lsprofcalltree import KCacheGrind
        d = {'func':func}
        prof = Profile()
        prof.runctx("res = func()", globals(), d)
        KCacheGrind(prof).output(open(goal + ".out", "w"))
        return d['res']

    def _do(self, goal, func, *args, **kwds):
        title = func.task_title
        if goal in self.done:
            self.log.info("already done: %s" % title)
            return
        else:
            self.log.info("%s..." % title)
        self.timer.start_event(goal)
        try:
            instrument = False
            try:
                if goal in PROFILE:
                    res = self._profile(goal, func)
                else:
                    res = func()
            except Instrument:
                instrument = True
            if not func.task_idempotent:
                self.done[goal] = True
            if instrument:
                self.proceed('compile')
                assert False, 'we should not get here'
        finally:
            self.timer.end_event(goal)
        return res

    def task_annotate(self):
        # includes annotation and annotatation simplifications
        translator = self.translator
        policy = self.policy
        self.log.info('with policy: %s.%s' % (policy.__class__.__module__, policy.__class__.__name__))

        annmodel.DEBUG = self.config.translation.debug
        annotator = translator.buildannotator(policy=policy)

        if self.entry_point:
            s = annotator.build_types(self.entry_point, self.inputtypes)

            self.sanity_check_annotation()
            if self.standalone and s.knowntype != int:
                raise Exception("stand-alone program entry point must return an "
                                "int (and not, e.g., None or always raise an "
                                "exception).")
            annotator.simplify()
            return s
        else:
            assert self.libdef is not None
            for func, inputtypes in self.libdef.functions:
                annotator.build_types(func, inputtypes)
            self.sanity_check_annotation()
            annotator.simplify()
    #
    task_annotate = taskdef(task_annotate, [], "Annotating&simplifying")


    def sanity_check_annotation(self):
        translator = self.translator
        irreg = query.qoutput(query.check_exceptblocks_qgen(translator))
        if irreg:
            self.log.info("Some exceptblocks seem insane")

        lost = query.qoutput(query.check_methods_qgen(translator))
        assert not lost, "lost methods, something gone wrong with the annotation of method defs"

        so = query.qoutput(query.polluted_qgen(translator))
        tot = len(translator.graphs)
        percent = int(tot and (100.0*so / tot) or 0)
        # if there are a few SomeObjects even if the policy doesn't allow
        # them, it means that they were put there in a controlled way
        # and then it's not a warning.
        if not translator.annotator.policy.allow_someobjects:
            pr = self.log.info
        elif percent == 0:
            pr = self.log.info
        else:
            pr = log.WARNING
        pr("-- someobjectness %2d%% (%d of %d functions polluted by SomeObjects)" % (percent, so, tot))



    def task_rtype_lltype(self):
        rtyper = self.translator.buildrtyper(type_system='lltype')
        insist = not self.config.translation.insist
        rtyper.specialize(dont_simplify_again=True,
                          crash_on_first_typeerror=insist)
    #
    task_rtype_lltype = taskdef(task_rtype_lltype, ['annotate'], "RTyping")
    RTYPE = 'rtype_lltype'

    def task_rtype_ootype(self):
        # Maybe type_system should simply be an option used in task_rtype
        insist = not self.config.translation.insist
        rtyper = self.translator.buildrtyper(type_system="ootype")
        rtyper.specialize(dont_simplify_again=True,
                          crash_on_first_typeerror=insist)
    #
    task_rtype_ootype = taskdef(task_rtype_ootype, ['annotate'], "ootyping")
    OOTYPE = 'rtype_ootype'

    def task_prehannotatebackendopt_lltype(self):
        from pypy.translator.backendopt.all import backend_optimizations
        backend_optimizations(self.translator,
                              inline_threshold=0,
                              merge_if_blocks=True,
                              constfold=True,
                              raisingop2direct_call=False,
                              remove_asserts=True)
    #
    task_prehannotatebackendopt_lltype = taskdef(
        task_prehannotatebackendopt_lltype,
        [RTYPE],
        "Backendopt before Hint-annotate")

    def task_hintannotate_lltype(self):
        from pypy.jit.hintannotator.annotator import HintAnnotator
        from pypy.jit.hintannotator.model import OriginFlags
        from pypy.jit.hintannotator.model import SomeLLAbstractConstant

        get_portal = self.extra['portal']
        PORTAL, POLICY = get_portal(self)
        t = self.translator
        self.portal_graph = graphof(t, PORTAL)

        hannotator = HintAnnotator(base_translator=t, policy=POLICY)
        self.hint_translator = hannotator.translator
        hs = hannotator.build_types(self.portal_graph,
                                    [SomeLLAbstractConstant(v.concretetype,
                                                            {OriginFlags(): True})
                                     for v in self.portal_graph.getargs()])
        count = hannotator.bookkeeper.nonstuboriggraphcount
        stubcount = hannotator.bookkeeper.stuboriggraphcount
        self.log.info("The hint-annotator saw %d graphs"
                      " (and made stubs for %d graphs)." % (count, stubcount))
        n = len(list(hannotator.translator.graphs[0].iterblocks()))
        self.log.info("portal has %d blocks" % n)
        self.hannotator = hannotator
    #
    task_hintannotate_lltype = taskdef(task_hintannotate_lltype,
                                       ['prehannotatebackendopt_lltype'],
                                       "Hint-annotate")

    def task_timeshift_lltype(self):
        from pypy.jit.timeshifter.hrtyper import HintRTyper
        from pypy.jit.codegen import detect_cpu
        cpu = detect_cpu.autodetect()
        if cpu == 'i386':
            from pypy.jit.codegen.i386.rgenop import RI386GenOp as RGenOp
            RGenOp.MC_SIZE = 32 * 1024 * 1024
        elif cpu == 'ppc':
            from pypy.jit.codegen.ppc.rgenop import RPPCGenOp as RGenOp
            RGenOp.MC_SIZE = 32 * 1024 * 1024
        else:
            raise Exception('Unsuported cpu %r'%cpu)

        del self.hint_translator
        ha = self.hannotator
        t = self.translator
        # make the timeshifted graphs
        hrtyper = HintRTyper(ha, t.rtyper, RGenOp)
        hrtyper.specialize(origportalgraph=self.portal_graph, view=False)
    #
    task_timeshift_lltype = taskdef(task_timeshift_lltype,
                             ["hintannotate_lltype"],
                             "Timeshift")

    def task_backendopt_lltype(self):
        from pypy.translator.backendopt.all import backend_optimizations
        backend_optimizations(self.translator)
    #
    task_backendopt_lltype = taskdef(task_backendopt_lltype,
                                     [RTYPE,
                                      '??timeshift_lltype'],
                                     "lltype back-end optimisations")
    BACKENDOPT = 'backendopt_lltype'

    def task_backendopt_ootype(self):
        from pypy.translator.backendopt.all import backend_optimizations
        backend_optimizations(self.translator)
    #
    task_backendopt_ootype = taskdef(task_backendopt_ootype, 
                                        [OOTYPE], "ootype back-end optimisations")
    OOBACKENDOPT = 'backendopt_ootype'


    def task_stackcheckinsertion_lltype(self):
        from pypy.translator.transform import insert_ll_stackcheck
        count = insert_ll_stackcheck(self.translator)
        self.log.info("inserted %d stack checks." % (count,))
        
    task_stackcheckinsertion_lltype = taskdef(
        task_stackcheckinsertion_lltype,
        ['?'+BACKENDOPT, RTYPE, 'annotate'],
        "inserting stack checks")
    STACKCHECKINSERTION = 'stackcheckinsertion_lltype'

    def task_database_c(self):
        translator = self.translator
        if translator.annotator is not None:
            translator.frozen = True

        standalone = self.standalone

        if standalone:
            from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
        else:
            from pypy.translator.c.genc import CExtModuleBuilder as CBuilder
        cbuilder = CBuilder(self.translator, self.entry_point,
                            config=self.config)
        cbuilder.stackless = self.config.translation.stackless
        if not standalone:     # xxx more messy
            cbuilder.modulename = self.extmod_name
        database = cbuilder.build_database()
        self.log.info("database for generating C source was created")
        self.cbuilder = cbuilder
        self.database = database
    #
    task_database_c = taskdef(task_database_c,
                            [STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE, '?annotate'], 
                            "Creating database for generating c source")
    
    def task_source_c(self):  # xxx messy
        translator = self.translator
        cbuilder = self.cbuilder
        database = self.database
        c_source_filename = cbuilder.generate_source(database)
        self.log.info("written: %s" % (c_source_filename,))
    #
    task_source_c = taskdef(task_source_c, ['database_c'], "Generating c source")

    def create_exe(self):
        if self.exe_name is not None:
            import shutil
            exename = mkexename(self.c_entryp)
            info = {'backend': self.config.translation.backend}
            newexename = self.exe_name % self.get_info()
            if '/' not in newexename and '\\' not in newexename:
                newexename = './' + newexename
            newexename = mkexename(newexename)
            shutil.copy(exename, newexename)
            self.c_entryp = newexename
        self.log.info("created: %s" % (self.c_entryp,))

    def task_compile_c(self): # xxx messy
        cbuilder = self.cbuilder
        cbuilder.compile()
        
        if self.standalone:
            self.c_entryp = cbuilder.executable_name
            self.create_exe()
        else:
            self.c_entryp = cbuilder.get_entry_point()
    #
    task_compile_c = taskdef(task_compile_c, ['source_c'], "Compiling c source")

    def backend_run(self, backend):
        c_entryp = self.c_entryp
        standalone = self.standalone 
        if standalone:
            os.system(c_entryp)
        else:
            runner = self.extra.get('run', lambda f: f())
            runner(c_entryp)

    def task_run_c(self):
        self.backend_run('c')
    #
    task_run_c = taskdef(task_run_c, ['compile_c'], 
                         "Running compiled c source",
                         idemp=True)

    def task_llinterpret_lltype(self):
        from pypy.rpython.llinterp import LLInterpreter
        py.log.setconsumer("llinterp operation", None)
        
        translator = self.translator
        interp = LLInterpreter(translator.rtyper)
        bk = translator.annotator.bookkeeper
        graph = bk.getdesc(self.entry_point).getuniquegraph()
        v = interp.eval_graph(graph,
                              self.extra.get('get_llinterp_args',
                                             lambda: [])())

        log.llinterpret.event("result -> %s" % v)
    #
    task_llinterpret_lltype = taskdef(task_llinterpret_lltype, 
                                      [STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE], 
                                      "LLInterpreting")

    def task_source_llvm(self):
        translator = self.translator
        if translator.annotator is None:
            raise ValueError, "llvm requires annotation."

        from pypy.translator.llvm import genllvm

        self.llvmgen = genllvm.GenLLVM(translator, self.standalone)

        llvm_filename = self.llvmgen.gen_source(self.entry_point)
        self.log.info("written: %s" % (llvm_filename,))
    #
    task_source_llvm = taskdef(task_source_llvm, 
                               [STACKCHECKINSERTION, BACKENDOPT, RTYPE], 
                               "Generating llvm source")

    def task_compile_llvm(self):
        gen = self.llvmgen
        if self.standalone:
            exe_name = (self.exe_name or 'testing') % self.get_info()
            self.c_entryp = gen.compile_standalone(exe_name)
            self.create_exe()
        else:
            self.c_module, self.c_entryp = gen.compile_module()
    #
    task_compile_llvm = taskdef(task_compile_llvm, 
                                ['source_llvm'], 
                                "Compiling llvm source")

    def task_run_llvm(self):
        self.backend_run('llvm')
    #
    task_run_llvm = taskdef(task_run_llvm, ['compile_llvm'], 
                            "Running compiled llvm source",
                            idemp=True)

    def task_source_js(self):
        from pypy.translator.js.js import JS
        self.gen = JS(self.translator, functions=[self.entry_point],
                      stackless=self.config.translation.stackless)
        filename = self.gen.write_source()
        self.log.info("Wrote %s" % (filename,))
    task_source_js = taskdef(task_source_js, 
                        [OOTYPE],
                        'Generating Javascript source')

    def task_compile_js(self):
        pass
    task_compile_js = taskdef(task_compile_js, ['source_js'],
                              'Skipping Javascript compilation')

    def task_run_js(self):
        pass
    task_run_js = taskdef(task_run_js, ['compile_js'],
                              'Please manually run the generated code')

    def task_source_cli(self):
        from pypy.translator.cli.gencli import GenCli
        from pypy.translator.cli.entrypoint import get_entrypoint

        if self.entry_point is not None: # executable mode
            entry_point_graph = self.translator.graphs[0]
            entry_point = get_entrypoint(entry_point_graph)
        else:
            # library mode
            assert self.libdef is not None
            bk = self.translator.annotator.bookkeeper
            entry_point = self.libdef.get_entrypoint(bk)

        self.gen = GenCli(udir, self.translator, entry_point, config=self.config)
        filename = self.gen.generate_source()
        self.log.info("Wrote %s" % (filename,))
    task_source_cli = taskdef(task_source_cli, ["?" + OOBACKENDOPT, OOTYPE],
                             'Generating CLI source')

    def task_compile_cli(self):
        from pypy.translator.oosupport.support import unpatch_os
        from pypy.translator.cli.test.runtest import CliFunctionWrapper
        filename = self.gen.build_exe()
        self.c_entryp = CliFunctionWrapper(filename)
        # restore original os values
        if hasattr(self, 'old_cli_defs'):
            unpatch_os(self.old_cli_defs)
        
        self.log.info("Compiled %s" % filename)
        if self.standalone and self.exe_name:
            self.copy_cli_exe()
    task_compile_cli = taskdef(task_compile_cli, ['source_cli'],
                              'Compiling CLI source')

    def copy_cli_exe(self):
        # XXX messy
        import os.path
        import shutil
        main_exe = self.c_entryp._exe
        usession_path, main_exe_name = os.path.split(main_exe)
        pypylib_dll = os.path.join(usession_path, 'pypylib.dll')

        basename = self.exe_name % self.get_info()
        dirname = basename + '-data/'
        if '/' not in dirname and '\\' not in dirname:
            dirname = './' + dirname

        if not os.path.exists(dirname):
            os.makedirs(dirname)
        shutil.copy(main_exe, dirname)
        shutil.copy(pypylib_dll, dirname)
        if bool(os.getenv('PYPY_GENCLI_COPYIL')):
            shutil.copy(os.path.join(usession_path, 'main.il'), dirname)
        newexename = basename
        f = file(newexename, 'w')
        f.write("""#!/bin/bash
LEDIT=`type -p ledit`
if [ `uname -s` = 'Cygwin' ]; then MONO=; else MONO=mono; fi
$LEDIT $MONO "$(dirname $0)/$(basename $0)-data/%s" "$@" # XXX doesn't work if it's placed in PATH
""" % main_exe_name)
        f.close()
        os.chmod(newexename, 0755)

    def copy_cli_dll(self):
        import os.path
        import shutil
        dllname = self.gen.outfile
        usession_path, dll_name = os.path.split(dllname)
        pypylib_dll = os.path.join(usession_path, 'pypylib.dll')
        shutil.copy(dllname, '.')
        shutil.copy(pypylib_dll, '.')
        
        # main.exe is a stub but is needed right now because it's
        # referenced by pypylib.dll.  Will be removed in the future
        translator_path, _ = os.path.split(__file__)
        main_exe = os.path.join(translator_path, 'cli/src/main.exe')
        shutil.copy(main_exe, '.')
        self.log.info("Copied to %s" % os.path.join(os.getcwd(), dllname))

    def task_run_cli(self):
        pass
    task_run_cli = taskdef(task_run_cli, ['compile_cli'],
                              'XXX')
    
    def task_source_jvm(self):
        from pypy.translator.jvm.genjvm import GenJvm
        from pypy.translator.jvm.node import EntryPoint

        entry_point_graph = self.translator.graphs[0]
        is_func = not self.standalone
        entry_point = EntryPoint(entry_point_graph, is_func, is_func)
        self.gen = GenJvm(udir, self.translator, entry_point)
        self.jvmsource = self.gen.generate_source()
        self.log.info("Wrote JVM code")
    task_source_jvm = taskdef(task_source_jvm, ["?" + OOBACKENDOPT, OOTYPE],
                             'Generating JVM source')

    def task_compile_jvm(self):
        from pypy.translator.oosupport.support import unpatch_os
        from pypy.translator.jvm.test.runtest import JvmGeneratedSourceWrapper
        self.jvmsource.compile()
        self.c_entryp = JvmGeneratedSourceWrapper(self.jvmsource)
        # restore original os values
        if hasattr(self, 'old_cli_defs'):
            unpatch_os(self.old_cli_defs)
        self.log.info("Compiled JVM source")
        if self.standalone and self.exe_name:
            self.copy_jvm_jar()
    task_compile_jvm = taskdef(task_compile_jvm, ['source_jvm'],
                              'Compiling JVM source')

    def copy_jvm_jar(self):
        from py.compat import subprocess
        basename = self.exe_name % self.get_info()
        root = udir.join('pypy')
        manifest = self.create_manifest(root)
        jnajar = py.path.local(__file__).dirpath('jvm', 'src', 'jna.jar')
        classlist = self.create_classlist(root, [jnajar])
        jarfile = py.path.local(basename + '.jar')
        self.log.info('Creating jar file')
        oldpath = root.chdir()
        subprocess.call(['jar', 'cmf', str(manifest), str(jarfile), '@'+str(classlist)])
        oldpath.chdir()

        # create a convenience script
        newexename = basename
        f = file(newexename, 'w')
        f.write("""#!/bin/bash
LEDIT=`type -p ledit`
$LEDIT java -Xmx256m -jar $0.jar "$@"
""")
        f.close()
        os.chmod(newexename, 0755)

    def create_manifest(self, root):
        filename = root.join('manifest.txt')
        manifest = filename.open('w')
        manifest.write('Main-class: pypy.Main\n\n')
        manifest.close()
        return filename

    def create_classlist(self, root, additional_jars=[]):
        from py.compat import subprocess
        # first, uncompress additional jars
        for jarfile in additional_jars:
            oldpwd = root.chdir()
            subprocess.call(['jar', 'xf', str(jarfile)])
            oldpwd.chdir()
        filename = root.join('classlist.txt')
        classlist = filename.open('w')
        classfiles = list(root.visit('*.class', True))
        classfiles += root.visit('*.so', True)
        classfiles += root.visit('*.dll', True)
        classfiles += root.visit('*.jnilib', True)
        for classfile in classfiles:
            print >> classlist, classfile.relto(root)
        classlist.close()
        return filename

    def task_run_jvm(self):
        pass
    task_run_jvm = taskdef(task_run_jvm, ['compile_jvm'],
                           'XXX')

    def proceed(self, goals):
        if not goals:
            if self.default_goal:
                goals = [self.default_goal]
            else:
                self.log.info("nothing to do")
                return
        elif isinstance(goals, str):
            goals = [goals]
        goals.extend(self.extra_goals)
        goals = self.backend_select_goals(goals)
        return self._execute(goals, task_skip = self._maybe_skip())

    def from_targetspec(targetspec_dic, config=None, args=None,
                        empty_translator=None,
                        disable=[],
                        default_goal=None):
        if args is None:
            args = []

        driver = TranslationDriver(config=config, default_goal=default_goal,
                                   disable=disable)
        # patch some attributes of the os module to make sure they
        # have the same value on every platform.
        backend, ts = driver.get_backend_and_type_system()
        if backend in ('cli', 'jvm'):
            from pypy.translator.oosupport.support import patch_os
            driver.old_cli_defs = patch_os()
        
        target = targetspec_dic['target']
        spec = target(driver, args)

        try:
            entry_point, inputtypes, policy = spec
        except ValueError:
            entry_point, inputtypes = spec
            policy = None

        driver.setup(entry_point, inputtypes, 
                     policy=policy, 
                     extra=targetspec_dic,
                     empty_translator=empty_translator)

        return driver

    from_targetspec = staticmethod(from_targetspec)

    def prereq_checkpt_rtype(self):
        assert 'pypy.rpython.rmodel' not in sys.modules, (
            "cannot fork because the rtyper has already been imported")
    prereq_checkpt_rtype_lltype = prereq_checkpt_rtype
    prereq_checkpt_rtype_ootype = prereq_checkpt_rtype    

    # checkpointing support
    def _event(self, kind, goal, func):
        if kind == 'pre':
            fork_before = self.config.translation.fork_before
            if fork_before:
                fork_before, = self.backend_select_goals([fork_before])
                if not fork_before in self.done and fork_before == goal:
                    prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
                    if prereq:
                        prereq()
                    from pypy.translator.goal import unixcheckpoint
                    unixcheckpoint.restartable_point(auto='run')
Exemple #7
0
 def task_source_js(self):
     from pypy.translator.js.js import JS
     self.gen = JS(self.translator, functions=[self.entry_point],
                   stackless=self.config.translation.stackless)
     filename = self.gen.write_source()
     self.log.info("Wrote %s" % (filename,))
Exemple #8
0
 def task_source_squeak(self):
     from pypy.translator.squeak.gensqueak import GenSqueak
     self.gen = GenSqueak(dir, self.translator)
     filename = self.gen.gen()
     self.log.info("Wrote %s" % (filename,))
Exemple #9
0
 def task_source_cl(self):
     from pypy.translator.lisp.gencl import GenCL
     self.gen = GenCL(self.translator, self.entry_point)
     filename = self.gen.emitfile()
     self.log.info("Wrote %s" % (filename,))
Exemple #10
0
class TranslationDriver(SimpleTaskEngine):
    _backend_extra_options = {}

    def __init__(
        self, setopts=None, default_goal=None, disable=[], exe_name=None, extmod_name=None, config=None, overrides=None
    ):
        self.timer = Timer()
        SimpleTaskEngine.__init__(self)

        self.log = log

        if config is None:
            from pypy.config.pypyoption import get_pypy_config

            config = get_pypy_config(translating=True)
        self.config = config
        if overrides is not None:
            self.config.override(overrides)

        if setopts is not None:
            self.config.set(**setopts)

        self.exe_name = exe_name
        self.extmod_name = extmod_name

        self.done = {}

        self.disable(disable)

        if default_goal:
            default_goal, = self.backend_select_goals([default_goal])
            if default_goal in self._maybe_skip():
                default_goal = None

        self.default_goal = default_goal
        self.extra_goals = []
        self.exposed = []

        # expose tasks
        def expose_task(task, backend_goal=None):
            if backend_goal is None:
                backend_goal = task

            def proc():
                return self.proceed(backend_goal)

            self.exposed.append(task)
            setattr(self, task, proc)

        backend, ts = self.get_backend_and_type_system()
        for task in self.tasks:
            explicit_task = task
            parts = task.split("_")
            if len(parts) == 1:
                if task in ("annotate",):
                    expose_task(task)
            else:
                task, postfix = parts
                if task in ("rtype", "backendopt", "llinterpret", "pyjitpl"):
                    if ts:
                        if ts == postfix:
                            expose_task(task, explicit_task)
                    else:
                        expose_task(explicit_task)
                elif task in ("source", "compile", "run"):
                    if backend:
                        if backend == postfix:
                            expose_task(task, explicit_task)
                    elif ts:
                        if ts == backend_to_typesystem(postfix):
                            expose_task(explicit_task)
                    else:
                        expose_task(explicit_task)

    def set_extra_goals(self, goals):
        self.extra_goals = goals

    def set_backend_extra_options(self, extra_options):
        self._backend_extra_options = extra_options

    def get_info(self):  # XXX more?
        d = {"backend": self.config.translation.backend}
        return d

    def get_backend_and_type_system(self):
        type_system = self.config.translation.type_system
        backend = self.config.translation.backend
        return backend, type_system

    def backend_select_goals(self, goals):
        backend, ts = self.get_backend_and_type_system()
        postfixes = [""] + ["_" + p for p in (backend, ts) if p]
        l = []
        for goal in goals:
            for postfix in postfixes:
                cand = "%s%s" % (goal, postfix)
                if cand in self.tasks:
                    new_goal = cand
                    break
            else:
                raise Exception, "cannot infer complete goal from: %r" % goal
            l.append(new_goal)
        return l

    def disable(self, to_disable):
        self._disabled = to_disable

    def _maybe_skip(self):
        maybe_skip = []
        if self._disabled:
            for goal in self.backend_select_goals(self._disabled):
                maybe_skip.extend(self._depending_on_closure(goal))
        return dict.fromkeys(maybe_skip).keys()

    def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
        standalone = inputtypes is None
        self.standalone = standalone

        if standalone:
            inputtypes = [s_list_of_strings]
        self.inputtypes = inputtypes

        if policy is None:
            policy = annpolicy.AnnotatorPolicy()
        if standalone:
            policy.allow_someobjects = False
        self.policy = policy

        self.extra = extra

        if empty_translator:
            translator = empty_translator
        else:
            translator = TranslationContext(config=self.config)

        self.entry_point = entry_point
        self.translator = translator
        self.libdef = None

        self.translator.driver_instrument_result = self.instrument_result

    def setup_library(self, libdef, policy=None, extra={}, empty_translator=None):
        self.setup(None, None, policy, extra, empty_translator)
        self.libdef = libdef

    def instrument_result(self, args):
        backend, ts = self.get_backend_and_type_system()
        if backend != "c" or sys.platform == "win32":
            raise Exception("instrumentation requires the c backend" " and unix for now")
        from pypy.tool.udir import udir

        datafile = udir.join("_instrument_counters")
        makeProfInstrument = lambda compiler: ProfInstrument(datafile, compiler)

        pid = os.fork()
        if pid == 0:
            # child compiling and running with instrumentation
            self.config.translation.instrument = True
            self.config.translation.instrumentctl = (makeProfInstrument, args)
            raise Instrument
        else:
            pid, status = os.waitpid(pid, 0)
            if os.WIFEXITED(status):
                status = os.WEXITSTATUS(status)
                if status != 0:
                    raise Exception, "instrumentation child failed: %d" % status
            else:
                raise Exception, "instrumentation child aborted"
            import array, struct

            n = datafile.size() // struct.calcsize("L")
            datafile = datafile.open("rb")
            counters = array.array("L")
            counters.fromfile(datafile, n)
            datafile.close()
            return counters

    def info(self, msg):
        log.info(msg)

    def _profile(self, goal, func):
        from cProfile import Profile
        from pypy.tool.lsprofcalltree import KCacheGrind

        d = {"func": func}
        prof = Profile()
        prof.runctx("res = func()", globals(), d)
        KCacheGrind(prof).output(open(goal + ".out", "w"))
        return d["res"]

    def _do(self, goal, func, *args, **kwds):
        title = func.task_title
        if goal in self.done:
            self.log.info("already done: %s" % title)
            return
        else:
            self.log.info("%s..." % title)
        self.timer.start_event(goal)
        try:
            instrument = False
            try:
                if goal in PROFILE:
                    res = self._profile(goal, func)
                else:
                    res = func()
            except Instrument:
                instrument = True
            if not func.task_idempotent:
                self.done[goal] = True
            if instrument:
                self.proceed("compile")
                assert False, "we should not get here"
        finally:
            try:
                self.timer.end_event(goal)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                pass
        return res

    def task_annotate(self):
        # includes annotation and annotatation simplifications
        translator = self.translator
        policy = self.policy
        self.log.info("with policy: %s.%s" % (policy.__class__.__module__, policy.__class__.__name__))

        annmodel.DEBUG = self.config.translation.debug
        annotator = translator.buildannotator(policy=policy)

        if self.entry_point:
            s = annotator.build_types(self.entry_point, self.inputtypes)

            self.sanity_check_annotation()
            if self.standalone and s.knowntype != int:
                raise Exception(
                    "stand-alone program entry point must return an "
                    "int (and not, e.g., None or always raise an "
                    "exception)."
                )
            annotator.simplify()
            return s
        else:
            assert self.libdef is not None
            for func, inputtypes in self.libdef.functions:
                annotator.build_types(func, inputtypes)
                func.c_name = func.func_name
            self.sanity_check_annotation()
            annotator.simplify()

    #
    task_annotate = taskdef(task_annotate, [], "Annotating&simplifying")

    def sanity_check_annotation(self):
        translator = self.translator
        irreg = query.qoutput(query.check_exceptblocks_qgen(translator))
        if irreg:
            self.log.info("Some exceptblocks seem insane")

        lost = query.qoutput(query.check_methods_qgen(translator))
        assert not lost, "lost methods, something gone wrong with the annotation of method defs"

        so = query.qoutput(query.polluted_qgen(translator))
        tot = len(translator.graphs)
        percent = int(tot and (100.0 * so / tot) or 0)
        # if there are a few SomeObjects even if the policy doesn't allow
        # them, it means that they were put there in a controlled way
        # and then it's not a warning.
        if not translator.annotator.policy.allow_someobjects:
            pr = self.log.info
        elif percent == 0:
            pr = self.log.info
        else:
            pr = log.WARNING
        pr("-- someobjectness %2d%% (%d of %d functions polluted by SomeObjects)" % (percent, so, tot))

    def task_rtype_lltype(self):
        rtyper = self.translator.buildrtyper(type_system="lltype")
        insist = not self.config.translation.insist
        rtyper.specialize(dont_simplify_again=True, crash_on_first_typeerror=insist)

    #
    task_rtype_lltype = taskdef(task_rtype_lltype, ["annotate"], "RTyping")
    RTYPE = "rtype_lltype"

    def task_rtype_ootype(self):
        # Maybe type_system should simply be an option used in task_rtype
        insist = not self.config.translation.insist
        rtyper = self.translator.buildrtyper(type_system="ootype")
        rtyper.specialize(dont_simplify_again=True, crash_on_first_typeerror=insist)

    #
    task_rtype_ootype = taskdef(task_rtype_ootype, ["annotate"], "ootyping")
    OOTYPE = "rtype_ootype"

    def task_pyjitpl_lltype(self):
        get_policy = self.extra["jitpolicy"]
        self.jitpolicy = get_policy(self)
        #
        from pypy.jit.metainterp.warmspot import apply_jit

        apply_jit(
            self.translator,
            policy=self.jitpolicy,
            debug_level=JIT_DEBUG[self.config.translation.jit_debug],
            backend_name=self.config.translation.jit_backend,
            inline=True,
        )
        #
        self.log.info("the JIT compiler was generated")

    #
    task_pyjitpl_lltype = taskdef(task_pyjitpl_lltype, [RTYPE], "JIT compiler generation")

    def task_pyjitpl_ootype(self):
        get_policy = self.extra["jitpolicy"]
        self.jitpolicy = get_policy(self)
        #
        from pypy.jit.metainterp.warmspot import apply_jit

        apply_jit(
            self.translator,
            policy=self.jitpolicy,
            debug_level=JIT_DEBUG[self.config.translation.jit_debug],
            backend_name="cli",
            inline=True,
        )  # XXX
        #
        self.log.info("the JIT compiler was generated")

    #
    task_pyjitpl_ootype = taskdef(task_pyjitpl_ootype, [OOTYPE], "JIT compiler generation")

    def task_backendopt_lltype(self):
        from pypy.translator.backendopt.all import backend_optimizations

        backend_optimizations(self.translator)

    #
    task_backendopt_lltype = taskdef(
        task_backendopt_lltype, [RTYPE, "??pyjitpl_lltype"], "lltype back-end optimisations"
    )
    BACKENDOPT = "backendopt_lltype"

    def task_backendopt_ootype(self):
        from pypy.translator.backendopt.all import backend_optimizations

        backend_optimizations(self.translator)

    #
    task_backendopt_ootype = taskdef(task_backendopt_ootype, [OOTYPE], "ootype back-end optimisations")
    OOBACKENDOPT = "backendopt_ootype"

    def task_stackcheckinsertion_lltype(self):
        from pypy.translator.transform import insert_ll_stackcheck

        count = insert_ll_stackcheck(self.translator)
        self.log.info("inserted %d stack checks." % (count,))

    task_stackcheckinsertion_lltype = taskdef(
        task_stackcheckinsertion_lltype, ["?" + BACKENDOPT, RTYPE, "annotate"], "inserting stack checks"
    )
    STACKCHECKINSERTION = "stackcheckinsertion_lltype"

    def possibly_check_for_boehm(self):
        if self.config.translation.gc == "boehm":
            from pypy.rpython.tool.rffi_platform import check_boehm

            if not check_boehm(self.translator.platform):
                i = 'Boehm GC not installed.  Try e.g. "translate.py --gc=hybrid"'
                raise Exception(i)

    def task_database_c(self):
        translator = self.translator
        if translator.annotator is not None:
            translator.frozen = True

        if self.libdef is not None:
            cbuilder = self.libdef.getcbuilder(self.translator, self.config)
            self.standalone = False
            standalone = False
        else:
            standalone = self.standalone

            if standalone:
                from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
            else:
                from pypy.translator.c.genc import CExtModuleBuilder as CBuilder
            cbuilder = CBuilder(self.translator, self.entry_point, config=self.config)
            cbuilder.stackless = self.config.translation.stackless
        if not standalone:  # xxx more messy
            cbuilder.modulename = self.extmod_name
        database = cbuilder.build_database()
        self.log.info("database for generating C source was created")
        self.cbuilder = cbuilder
        self.database = database

    #
    task_database_c = taskdef(
        task_database_c,
        [STACKCHECKINSERTION, "?" + BACKENDOPT, RTYPE, "?annotate"],
        "Creating database for generating c source",
        earlycheck=possibly_check_for_boehm,
    )

    def task_source_c(self):  # xxx messy
        translator = self.translator
        cbuilder = self.cbuilder
        database = self.database
        if self._backend_extra_options.get("c_debug_defines", False):
            defines = cbuilder.DEBUG_DEFINES
        else:
            defines = {}
        c_source_filename = cbuilder.generate_source(database, defines)
        self.log.info("written: %s" % (c_source_filename,))
        if self.config.translation.dump_static_data_info:
            from pypy.translator.tool.staticsizereport import dump_static_data_info

            targetdir = cbuilder.targetdir
            fname = dump_static_data_info(self.log, database, targetdir)
            dstname = self.compute_exe_name() + ".staticdata.info"
            shutil.copy(str(fname), str(dstname))
            self.log.info("Static data info written to %s" % dstname)

    #
    task_source_c = taskdef(task_source_c, ["database_c"], "Generating c source")

    def compute_exe_name(self):
        newexename = self.exe_name % self.get_info()
        if "/" not in newexename and "\\" not in newexename:
            newexename = "./" + newexename
        return mkexename(py.path.local(newexename))

    def create_exe(self):
        if self.exe_name is not None:
            exename = mkexename(self.c_entryp)
            newexename = self.compute_exe_name()
            shutil.copy(str(exename), str(newexename))
            self.c_entryp = newexename
        self.log.info("created: %s" % (self.c_entryp,))

    def task_compile_c(self):  # xxx messy
        cbuilder = self.cbuilder
        cbuilder.compile()

        if self.standalone:
            self.c_entryp = cbuilder.executable_name
            self.create_exe()
        else:
            isolated = self._backend_extra_options.get("c_isolated", False)
            self.c_entryp = cbuilder.get_entry_point(isolated=isolated)

    #
    task_compile_c = taskdef(task_compile_c, ["source_c"], "Compiling c source")

    def backend_run(self, backend):
        c_entryp = self.c_entryp
        standalone = self.standalone
        if standalone:
            os.system(c_entryp)
        else:
            runner = self.extra.get("run", lambda f: f())
            runner(c_entryp)

    def task_run_c(self):
        self.backend_run("c")

    #
    task_run_c = taskdef(task_run_c, ["compile_c"], "Running compiled c source", idemp=True)

    def task_llinterpret_lltype(self):
        from pypy.rpython.llinterp import LLInterpreter

        py.log.setconsumer("llinterp operation", None)

        translator = self.translator
        interp = LLInterpreter(translator.rtyper)
        bk = translator.annotator.bookkeeper
        graph = bk.getdesc(self.entry_point).getuniquegraph()
        v = interp.eval_graph(graph, self.extra.get("get_llinterp_args", lambda: [])())

        log.llinterpret.event("result -> %s" % v)

    #
    task_llinterpret_lltype = taskdef(
        task_llinterpret_lltype, [STACKCHECKINSERTION, "?" + BACKENDOPT, RTYPE], "LLInterpreting"
    )

    def task_source_cli(self):
        from pypy.translator.cli.gencli import GenCli
        from pypy.translator.cli.entrypoint import get_entrypoint

        if self.entry_point is not None:  # executable mode
            entry_point_graph = self.translator.graphs[0]
            entry_point = get_entrypoint(entry_point_graph)
        else:
            # library mode
            assert self.libdef is not None
            bk = self.translator.annotator.bookkeeper
            entry_point = self.libdef.get_entrypoint(bk)

        self.gen = GenCli(udir, self.translator, entry_point, config=self.config)
        filename = self.gen.generate_source()
        self.log.info("Wrote %s" % (filename,))

    task_source_cli = taskdef(task_source_cli, ["?" + OOBACKENDOPT, OOTYPE], "Generating CLI source")

    def task_compile_cli(self):
        from pypy.translator.oosupport.support import unpatch_os
        from pypy.translator.cli.test.runtest import CliFunctionWrapper

        filename = self.gen.build_exe()
        self.c_entryp = CliFunctionWrapper(filename)
        # restore original os values
        if hasattr(self, "old_cli_defs"):
            unpatch_os(self.old_cli_defs)

        self.log.info("Compiled %s" % filename)
        if self.standalone and self.exe_name:
            self.copy_cli_exe()

    task_compile_cli = taskdef(task_compile_cli, ["source_cli"], "Compiling CLI source")

    def copy_cli_exe(self):
        # XXX messy
        main_exe = self.c_entryp._exe
        usession_path, main_exe_name = os.path.split(main_exe)
        pypylib_dll = os.path.join(usession_path, "pypylib.dll")

        basename = self.exe_name % self.get_info()
        dirname = basename + "-data/"
        if "/" not in dirname and "\\" not in dirname:
            dirname = "./" + dirname

        if not os.path.exists(dirname):
            os.makedirs(dirname)
        shutil.copy(main_exe, dirname)
        shutil.copy(pypylib_dll, dirname)
        if bool(os.getenv("PYPY_GENCLI_COPYIL")):
            shutil.copy(os.path.join(usession_path, "main.il"), dirname)
        newexename = basename
        f = file(newexename, "w")
        f.write(
            """#!/bin/bash
LEDIT=`type -p ledit`
EXE=`readlink $0`
if [ -z $EXE ]
then
    EXE=$0
fi
if  uname -s | grep -iq Cygwin ; then MONO=; else MONO=mono; fi
$LEDIT $MONO "$(dirname $EXE)/$(basename $EXE)-data/%s" "$@" # XXX doesn't work if it's placed in PATH
"""
            % main_exe_name
        )
        f.close()
        os.chmod(newexename, 0755)

    def copy_cli_dll(self):
        dllname = self.gen.outfile
        usession_path, dll_name = os.path.split(dllname)
        pypylib_dll = os.path.join(usession_path, "pypylib.dll")
        shutil.copy(dllname, ".")
        shutil.copy(pypylib_dll, ".")

        # main.exe is a stub but is needed right now because it's
        # referenced by pypylib.dll.  Will be removed in the future
        translator_path, _ = os.path.split(__file__)
        main_exe = os.path.join(translator_path, "cli/src/main.exe")
        shutil.copy(main_exe, ".")
        self.log.info("Copied to %s" % os.path.join(os.getcwd(), dllname))

    def task_run_cli(self):
        pass

    task_run_cli = taskdef(task_run_cli, ["compile_cli"], "XXX")

    def task_source_jvm(self):
        from pypy.translator.jvm.genjvm import GenJvm
        from pypy.translator.jvm.node import EntryPoint

        entry_point_graph = self.translator.graphs[0]
        is_func = not self.standalone
        entry_point = EntryPoint(entry_point_graph, is_func, is_func)
        self.gen = GenJvm(udir, self.translator, entry_point)
        self.jvmsource = self.gen.generate_source()
        self.log.info("Wrote JVM code")

    task_source_jvm = taskdef(task_source_jvm, ["?" + OOBACKENDOPT, OOTYPE], "Generating JVM source")

    def task_compile_jvm(self):
        from pypy.translator.oosupport.support import unpatch_os
        from pypy.translator.jvm.test.runtest import JvmGeneratedSourceWrapper

        self.jvmsource.compile()
        self.c_entryp = JvmGeneratedSourceWrapper(self.jvmsource)
        # restore original os values
        if hasattr(self, "old_cli_defs"):
            unpatch_os(self.old_cli_defs)
        self.log.info("Compiled JVM source")
        if self.standalone and self.exe_name:
            self.copy_jvm_jar()

    task_compile_jvm = taskdef(task_compile_jvm, ["source_jvm"], "Compiling JVM source")

    def copy_jvm_jar(self):
        from py.compat import subprocess

        basename = self.exe_name % self.get_info()
        root = udir.join("pypy")
        manifest = self.create_manifest(root)
        jnajar = py.path.local(__file__).dirpath("jvm", "src", "jna.jar")
        classlist = self.create_classlist(root, [jnajar])
        jarfile = py.path.local(basename + ".jar")
        self.log.info("Creating jar file")
        oldpath = root.chdir()
        subprocess.call(["jar", "cmf", str(manifest), str(jarfile), "@" + str(classlist)])
        oldpath.chdir()

        # create a convenience script
        newexename = basename
        f = file(newexename, "w")
        f.write(
            """#!/bin/bash
LEDIT=`type -p ledit`
EXE=`readlink $0`
if [ -z $EXE ]
then
    EXE=$0
fi
$LEDIT java -Xmx256m -jar $EXE.jar "$@"
"""
        )
        f.close()
        os.chmod(newexename, 0755)

    def create_manifest(self, root):
        filename = root.join("manifest.txt")
        manifest = filename.open("w")
        manifest.write("Main-class: pypy.Main\n\n")
        manifest.close()
        return filename

    def create_classlist(self, root, additional_jars=[]):
        from py.compat import subprocess

        # first, uncompress additional jars
        for jarfile in additional_jars:
            oldpwd = root.chdir()
            subprocess.call(["jar", "xf", str(jarfile)])
            oldpwd.chdir()
        filename = root.join("classlist.txt")
        classlist = filename.open("w")
        classfiles = list(root.visit("*.class", True))
        classfiles += root.visit("*.so", True)
        classfiles += root.visit("*.dll", True)
        classfiles += root.visit("*.jnilib", True)
        for classfile in classfiles:
            print >> classlist, classfile.relto(root)
        classlist.close()
        return filename

    def task_run_jvm(self):
        pass

    task_run_jvm = taskdef(task_run_jvm, ["compile_jvm"], "XXX")

    def proceed(self, goals):
        if not goals:
            if self.default_goal:
                goals = [self.default_goal]
            else:
                self.log.info("nothing to do")
                return
        elif isinstance(goals, str):
            goals = [goals]
        goals.extend(self.extra_goals)
        goals = self.backend_select_goals(goals)
        return self._execute(goals, task_skip=self._maybe_skip())

    def from_targetspec(targetspec_dic, config=None, args=None, empty_translator=None, disable=[], default_goal=None):
        if args is None:
            args = []

        driver = TranslationDriver(config=config, default_goal=default_goal, disable=disable)
        # patch some attributes of the os module to make sure they
        # have the same value on every platform.
        backend, ts = driver.get_backend_and_type_system()
        if backend in ("cli", "jvm"):
            from pypy.translator.oosupport.support import patch_os

            driver.old_cli_defs = patch_os()

        target = targetspec_dic["target"]
        spec = target(driver, args)

        try:
            entry_point, inputtypes, policy = spec
        except ValueError:
            entry_point, inputtypes = spec
            policy = None

        driver.setup(entry_point, inputtypes, policy=policy, extra=targetspec_dic, empty_translator=empty_translator)

        return driver

    from_targetspec = staticmethod(from_targetspec)

    def prereq_checkpt_rtype(self):
        assert "pypy.rpython.rmodel" not in sys.modules, "cannot fork because the rtyper has already been imported"

    prereq_checkpt_rtype_lltype = prereq_checkpt_rtype
    prereq_checkpt_rtype_ootype = prereq_checkpt_rtype

    # checkpointing support
    def _event(self, kind, goal, func):
        if kind == "planned" and func.task_earlycheck:
            func.task_earlycheck(self)
        if kind == "pre":
            fork_before = self.config.translation.fork_before
            if fork_before:
                fork_before, = self.backend_select_goals([fork_before])
                if not fork_before in self.done and fork_before == goal:
                    prereq = getattr(self, "prereq_checkpt_%s" % goal, None)
                    if prereq:
                        prereq()
                    from pypy.translator.goal import unixcheckpoint

                    unixcheckpoint.restartable_point(auto="run")
Exemple #11
0
class TranslationDriver(SimpleTaskEngine):
    _backend_extra_options = {}

    def __init__(self, setopts=None, default_goal=None,
                 disable=[],
                 exe_name=None, extmod_name=None,
                 config=None, overrides=None):
        self.timer = Timer()
        SimpleTaskEngine.__init__(self)

        self.log = log

        if config is None:
            from pypy.config.pypyoption import get_pypy_config
            config = get_pypy_config(translating=True)
        self.config = config
        if overrides is not None:
            self.config.override(overrides)

        if setopts is not None:
            self.config.set(**setopts)
        
        self.exe_name = exe_name
        self.extmod_name = extmod_name

        self.done = {}

        self.disable(disable)

        if default_goal:
            default_goal, = self.backend_select_goals([default_goal])
            if default_goal in self._maybe_skip():
                default_goal = None
        
        self.default_goal = default_goal
        self.extra_goals = []
        self.exposed = []

        # expose tasks
        def expose_task(task, backend_goal=None):
            if backend_goal is None:
                backend_goal = task
            def proc():
                return self.proceed(backend_goal)
            self.exposed.append(task)
            setattr(self, task, proc)

        backend, ts = self.get_backend_and_type_system()
        for task in self.tasks:
            explicit_task = task
            parts = task.split('_')
            if len(parts) == 1:
                if task in ('annotate',):
                    expose_task(task)
            else:
                task, postfix = parts
                if task in ('rtype', 'backendopt', 'llinterpret',
                            'pyjitpl'):
                    if ts:
                        if ts == postfix:
                            expose_task(task, explicit_task)
                    else:
                        expose_task(explicit_task)
                elif task in ('source', 'compile', 'run'):
                    if backend:
                        if backend == postfix:
                            expose_task(task, explicit_task)
                    elif ts:
                        if ts == backend_to_typesystem(postfix):
                            expose_task(explicit_task)
                    else:
                        expose_task(explicit_task)

    def set_extra_goals(self, goals):
        self.extra_goals = goals

    def set_backend_extra_options(self, extra_options):
        self._backend_extra_options = extra_options
        
    def get_info(self): # XXX more?
        d = {'backend': self.config.translation.backend}
        return d

    def get_backend_and_type_system(self):
        type_system = self.config.translation.type_system
        backend = self.config.translation.backend
        return backend, type_system

    def backend_select_goals(self, goals):
        backend, ts = self.get_backend_and_type_system()
        postfixes = [''] + ['_'+p for p in (backend, ts) if p]
        l = []
        for goal in goals:
            for postfix in postfixes:
                cand = "%s%s" % (goal, postfix)
                if cand in self.tasks:
                    new_goal = cand
                    break
            else:
                raise Exception, "cannot infer complete goal from: %r" % goal 
            l.append(new_goal)
        return l

    def disable(self, to_disable):
        self._disabled = to_disable

    def _maybe_skip(self):
        maybe_skip = []
        if self._disabled:
             for goal in  self.backend_select_goals(self._disabled):
                 maybe_skip.extend(self._depending_on_closure(goal))
        return dict.fromkeys(maybe_skip).keys()


    def setup(self, entry_point, inputtypes, policy=None, extra={}, empty_translator=None):
        standalone = inputtypes is None
        self.standalone = standalone

        if standalone:
            inputtypes = [s_list_of_strings]
        self.inputtypes = inputtypes

        if policy is None:
            policy = annpolicy.AnnotatorPolicy()
        if standalone:
            policy.allow_someobjects = False
        self.policy = policy

        self.extra = extra

        if empty_translator:
            translator = empty_translator
        else:
            translator = TranslationContext(config=self.config)

        self.entry_point = entry_point
        self.translator = translator
        self.libdef = None

        self.translator.driver_instrument_result = self.instrument_result

    def setup_library(self, libdef, policy=None, extra={}, empty_translator=None):
        self.setup(None, None, policy, extra, empty_translator)
        self.libdef = libdef

    def instrument_result(self, args):
        backend, ts = self.get_backend_and_type_system()
        if backend != 'c' or sys.platform == 'win32':
            raise Exception("instrumentation requires the c backend"
                            " and unix for now")
        from pypy.tool.udir import udir
        
        datafile = udir.join('_instrument_counters')
        makeProfInstrument = lambda compiler: ProfInstrument(datafile, compiler)

        pid = os.fork()
        if pid == 0:
            # child compiling and running with instrumentation
            self.config.translation.instrument = True
            self.config.translation.instrumentctl = (makeProfInstrument,
                                                     args)
            raise Instrument
        else:
            pid, status = os.waitpid(pid, 0)
            if os.WIFEXITED(status):
                status = os.WEXITSTATUS(status)
                if status != 0:
                    raise Exception, "instrumentation child failed: %d" % status
            else:
                raise Exception, "instrumentation child aborted"
            import array, struct
            n = datafile.size()//struct.calcsize('L')
            datafile = datafile.open('rb')
            counters = array.array('L')
            counters.fromfile(datafile, n)
            datafile.close()
            return counters

    def info(self, msg):
        log.info(msg)

    def _profile(self, goal, func):
        from cProfile import Profile
        from pypy.tool.lsprofcalltree import KCacheGrind
        d = {'func':func}
        prof = Profile()
        prof.runctx("res = func()", globals(), d)
        KCacheGrind(prof).output(open(goal + ".out", "w"))
        return d['res']

    def _do(self, goal, func, *args, **kwds):
        title = func.task_title
        if goal in self.done:
            self.log.info("already done: %s" % title)
            return
        else:
            self.log.info("%s..." % title)
        self.timer.start_event(goal)
        try:
            instrument = False
            try:
                if goal in PROFILE:
                    res = self._profile(goal, func)
                else:
                    res = func()
            except Instrument:
                instrument = True
            if not func.task_idempotent:
                self.done[goal] = True
            if instrument:
                self.proceed('compile')
                assert False, 'we should not get here'
        finally:
            try:
                self.timer.end_event(goal)
            except (KeyboardInterrupt, SystemExit):
                raise
            except:
                pass
        return res

    def task_annotate(self):
        # includes annotation and annotatation simplifications
        translator = self.translator
        policy = self.policy
        self.log.info('with policy: %s.%s' % (policy.__class__.__module__, policy.__class__.__name__))

        annmodel.DEBUG = self.config.translation.debug
        annotator = translator.buildannotator(policy=policy)

        if self.entry_point:
            s = annotator.build_types(self.entry_point, self.inputtypes)

            self.sanity_check_annotation()
            if self.standalone and s.knowntype != int:
                raise Exception("stand-alone program entry point must return an "
                                "int (and not, e.g., None or always raise an "
                                "exception).")
            annotator.simplify()
            return s
        else:
            assert self.libdef is not None
            for func, inputtypes in self.libdef.functions:
                annotator.build_types(func, inputtypes)
                func.c_name = func.func_name
            self.sanity_check_annotation()
            annotator.simplify()
    #
    task_annotate = taskdef(task_annotate, [], "Annotating&simplifying")


    def sanity_check_annotation(self):
        translator = self.translator
        irreg = query.qoutput(query.check_exceptblocks_qgen(translator))
        if irreg:
            self.log.info("Some exceptblocks seem insane")

        lost = query.qoutput(query.check_methods_qgen(translator))
        assert not lost, "lost methods, something gone wrong with the annotation of method defs"

        so = query.qoutput(query.polluted_qgen(translator))
        tot = len(translator.graphs)
        percent = int(tot and (100.0*so / tot) or 0)
        # if there are a few SomeObjects even if the policy doesn't allow
        # them, it means that they were put there in a controlled way
        # and then it's not a warning.
        if not translator.annotator.policy.allow_someobjects:
            pr = self.log.info
        elif percent == 0:
            pr = self.log.info
        else:
            pr = log.WARNING
        pr("-- someobjectness %2d%% (%d of %d functions polluted by SomeObjects)" % (percent, so, tot))



    def task_rtype_lltype(self):
        rtyper = self.translator.buildrtyper(type_system='lltype')
        insist = not self.config.translation.insist
        rtyper.specialize(dont_simplify_again=True,
                          crash_on_first_typeerror=insist)
    #
    task_rtype_lltype = taskdef(task_rtype_lltype, ['annotate'], "RTyping")
    RTYPE = 'rtype_lltype'

    def task_rtype_ootype(self):
        # Maybe type_system should simply be an option used in task_rtype
        insist = not self.config.translation.insist
        rtyper = self.translator.buildrtyper(type_system="ootype")
        rtyper.specialize(dont_simplify_again=True,
                          crash_on_first_typeerror=insist)
    #
    task_rtype_ootype = taskdef(task_rtype_ootype, ['annotate'], "ootyping")
    OOTYPE = 'rtype_ootype'

    def task_pyjitpl_lltype(self):
        get_policy = self.extra['jitpolicy']
        self.jitpolicy = get_policy(self)
        #
        from pypy.jit.metainterp.warmspot import apply_jit
        apply_jit(self.translator, policy=self.jitpolicy,
                  debug_level=JIT_DEBUG[self.config.translation.jit_debug],
                  backend_name=self.config.translation.jit_backend, inline=True)
        #
        self.log.info("the JIT compiler was generated")
    #
    task_pyjitpl_lltype = taskdef(task_pyjitpl_lltype,
                                  [RTYPE],
                                  "JIT compiler generation")

    def task_pyjitpl_ootype(self):
        get_policy = self.extra['jitpolicy']
        self.jitpolicy = get_policy(self)
        #
        from pypy.jit.metainterp.warmspot import apply_jit
        apply_jit(self.translator, policy=self.jitpolicy,
                  debug_level=JIT_DEBUG[self.config.translation.jit_debug],
                  backend_name='cli', inline=True) #XXX
        #
        self.log.info("the JIT compiler was generated")
    #
    task_pyjitpl_ootype = taskdef(task_pyjitpl_ootype,
                                  [OOTYPE],
                                  "JIT compiler generation")

    def task_backendopt_lltype(self):
        from pypy.translator.backendopt.all import backend_optimizations
        backend_optimizations(self.translator)
    #
    task_backendopt_lltype = taskdef(task_backendopt_lltype,
                                     [RTYPE, '??pyjitpl_lltype'],
                                     "lltype back-end optimisations")
    BACKENDOPT = 'backendopt_lltype'

    def task_backendopt_ootype(self):
        from pypy.translator.backendopt.all import backend_optimizations
        backend_optimizations(self.translator)
    #
    task_backendopt_ootype = taskdef(task_backendopt_ootype, 
                                        [OOTYPE], "ootype back-end optimisations")
    OOBACKENDOPT = 'backendopt_ootype'


    def task_stackcheckinsertion_lltype(self):
        from pypy.translator.transform import insert_ll_stackcheck
        count = insert_ll_stackcheck(self.translator)
        self.log.info("inserted %d stack checks." % (count,))
        
    task_stackcheckinsertion_lltype = taskdef(
        task_stackcheckinsertion_lltype,
        ['?'+BACKENDOPT, RTYPE, 'annotate'],
        "inserting stack checks")
    STACKCHECKINSERTION = 'stackcheckinsertion_lltype'

    def possibly_check_for_boehm(self):
        if self.config.translation.gc == "boehm":
            from pypy.rpython.tool.rffi_platform import check_boehm
            if not check_boehm(self.translator.platform):
                i = 'Boehm GC not installed.  Try e.g. "translate.py --gc=hybrid"'
                raise Exception(i)

    def task_database_c(self):
        translator = self.translator
        if translator.annotator is not None:
            translator.frozen = True

        if self.libdef is not None:
            cbuilder = self.libdef.getcbuilder(self.translator, self.config)
            self.standalone = False
            standalone = False
        else:
            standalone = self.standalone

            if standalone:
                from pypy.translator.c.genc import CStandaloneBuilder as CBuilder
            else:
                from pypy.translator.c.genc import CExtModuleBuilder as CBuilder
            cbuilder = CBuilder(self.translator, self.entry_point,
                                config=self.config)
            cbuilder.stackless = self.config.translation.stackless
        if not standalone:     # xxx more messy
            cbuilder.modulename = self.extmod_name
        database = cbuilder.build_database()
        self.log.info("database for generating C source was created")
        self.cbuilder = cbuilder
        self.database = database
    #
    task_database_c = taskdef(task_database_c,
                            [STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE, '?annotate'], 
                            "Creating database for generating c source",
                            earlycheck = possibly_check_for_boehm)
    
    def task_source_c(self):  # xxx messy
        translator = self.translator
        cbuilder = self.cbuilder
        database = self.database
        if self._backend_extra_options.get('c_debug_defines', False):
            defines = cbuilder.DEBUG_DEFINES
        else:
            defines = {}
        c_source_filename = cbuilder.generate_source(database, defines)
        self.log.info("written: %s" % (c_source_filename,))
        if self.config.translation.dump_static_data_info:
            from pypy.translator.tool.staticsizereport import dump_static_data_info
            targetdir = cbuilder.targetdir
            fname = dump_static_data_info(self.log, database, targetdir)
            dstname = self.compute_exe_name() + '.staticdata.info'
            shutil.copy(str(fname), str(dstname))
            self.log.info('Static data info written to %s' % dstname)

    #
    task_source_c = taskdef(task_source_c, ['database_c'], "Generating c source")

    def compute_exe_name(self):
        newexename = self.exe_name % self.get_info()
        if '/' not in newexename and '\\' not in newexename:
            newexename = './' + newexename
        return mkexename(py.path.local(newexename))

    def create_exe(self):
        if self.exe_name is not None:
            exename = mkexename(self.c_entryp)
            newexename = self.compute_exe_name()
            shutil.copy(str(exename), str(newexename))
            self.c_entryp = newexename
        self.log.info("created: %s" % (self.c_entryp,))

    def task_compile_c(self): # xxx messy
        cbuilder = self.cbuilder
        cbuilder.compile()

        if self.standalone:
            self.c_entryp = cbuilder.executable_name
            self.create_exe()
        else:
            isolated = self._backend_extra_options.get('c_isolated', False)
            self.c_entryp = cbuilder.get_entry_point(isolated=isolated)
    #
    task_compile_c = taskdef(task_compile_c, ['source_c'], "Compiling c source")

    def backend_run(self, backend):
        c_entryp = self.c_entryp
        standalone = self.standalone 
        if standalone:
            os.system(c_entryp)
        else:
            runner = self.extra.get('run', lambda f: f())
            runner(c_entryp)

    def task_run_c(self):
        self.backend_run('c')
    #
    task_run_c = taskdef(task_run_c, ['compile_c'], 
                         "Running compiled c source",
                         idemp=True)

    def task_llinterpret_lltype(self):
        from pypy.rpython.llinterp import LLInterpreter
        py.log.setconsumer("llinterp operation", None)
        
        translator = self.translator
        interp = LLInterpreter(translator.rtyper)
        bk = translator.annotator.bookkeeper
        graph = bk.getdesc(self.entry_point).getuniquegraph()
        v = interp.eval_graph(graph,
                              self.extra.get('get_llinterp_args',
                                             lambda: [])())

        log.llinterpret.event("result -> %s" % v)
    #
    task_llinterpret_lltype = taskdef(task_llinterpret_lltype, 
                                      [STACKCHECKINSERTION, '?'+BACKENDOPT, RTYPE], 
                                      "LLInterpreting")

    def task_source_cli(self):
        from pypy.translator.cli.gencli import GenCli
        from pypy.translator.cli.entrypoint import get_entrypoint

        if self.entry_point is not None: # executable mode
            entry_point_graph = self.translator.graphs[0]
            entry_point = get_entrypoint(entry_point_graph)
        else:
            # library mode
            assert self.libdef is not None
            bk = self.translator.annotator.bookkeeper
            entry_point = self.libdef.get_entrypoint(bk)

        self.gen = GenCli(udir, self.translator, entry_point, config=self.config)
        filename = self.gen.generate_source()
        self.log.info("Wrote %s" % (filename,))
    task_source_cli = taskdef(task_source_cli, ["?" + OOBACKENDOPT, OOTYPE],
                             'Generating CLI source')

    def task_compile_cli(self):
        from pypy.translator.oosupport.support import unpatch_os
        from pypy.translator.cli.test.runtest import CliFunctionWrapper
        filename = self.gen.build_exe()
        self.c_entryp = CliFunctionWrapper(filename)
        # restore original os values
        if hasattr(self, 'old_cli_defs'):
            unpatch_os(self.old_cli_defs)
        
        self.log.info("Compiled %s" % filename)
        if self.standalone and self.exe_name:
            self.copy_cli_exe()
    task_compile_cli = taskdef(task_compile_cli, ['source_cli'],
                              'Compiling CLI source')

    def copy_cli_exe(self):
        # XXX messy
        main_exe = self.c_entryp._exe
        usession_path, main_exe_name = os.path.split(main_exe)
        pypylib_dll = os.path.join(usession_path, 'pypylib.dll')

        basename = self.exe_name % self.get_info()
        dirname = basename + '-data/'
        if '/' not in dirname and '\\' not in dirname:
            dirname = './' + dirname

        if not os.path.exists(dirname):
            os.makedirs(dirname)
        shutil.copy(main_exe, dirname)
        shutil.copy(pypylib_dll, dirname)
        if bool(os.getenv('PYPY_GENCLI_COPYIL')):
            shutil.copy(os.path.join(usession_path, 'main.il'), dirname)
        newexename = basename
        f = file(newexename, 'w')
        f.write("""#!/bin/bash
LEDIT=`type -p ledit`
EXE=`readlink $0`
if [ -z $EXE ]
then
    EXE=$0
fi
if  uname -s | grep -iq Cygwin ; then MONO=; else MONO=mono; fi
$LEDIT $MONO "$(dirname $EXE)/$(basename $EXE)-data/%s" "$@" # XXX doesn't work if it's placed in PATH
""" % main_exe_name)
        f.close()
        os.chmod(newexename, 0755)

    def copy_cli_dll(self):
        dllname = self.gen.outfile
        usession_path, dll_name = os.path.split(dllname)
        pypylib_dll = os.path.join(usession_path, 'pypylib.dll')
        shutil.copy(dllname, '.')
        shutil.copy(pypylib_dll, '.')
        
        # main.exe is a stub but is needed right now because it's
        # referenced by pypylib.dll.  Will be removed in the future
        translator_path, _ = os.path.split(__file__)
        main_exe = os.path.join(translator_path, 'cli/src/main.exe')
        shutil.copy(main_exe, '.')
        self.log.info("Copied to %s" % os.path.join(os.getcwd(), dllname))

    def task_run_cli(self):
        pass
    task_run_cli = taskdef(task_run_cli, ['compile_cli'],
                              'XXX')
    
    def task_source_jvm(self):
        from pypy.translator.jvm.genjvm import GenJvm
        from pypy.translator.jvm.node import EntryPoint

        entry_point_graph = self.translator.graphs[0]
        is_func = not self.standalone
        entry_point = EntryPoint(entry_point_graph, is_func, is_func)
        self.gen = GenJvm(udir, self.translator, entry_point)
        self.jvmsource = self.gen.generate_source()
        self.log.info("Wrote JVM code")
    task_source_jvm = taskdef(task_source_jvm, ["?" + OOBACKENDOPT, OOTYPE],
                             'Generating JVM source')

    def task_compile_jvm(self):
        from pypy.translator.oosupport.support import unpatch_os
        from pypy.translator.jvm.test.runtest import JvmGeneratedSourceWrapper
        self.jvmsource.compile()
        self.c_entryp = JvmGeneratedSourceWrapper(self.jvmsource)
        # restore original os values
        if hasattr(self, 'old_cli_defs'):
            unpatch_os(self.old_cli_defs)
        self.log.info("Compiled JVM source")
        if self.standalone and self.exe_name:
            self.copy_jvm_jar()
    task_compile_jvm = taskdef(task_compile_jvm, ['source_jvm'],
                              'Compiling JVM source')

    def copy_jvm_jar(self):
        from py.compat import subprocess
        basename = self.exe_name % self.get_info()
        root = udir.join('pypy')
        manifest = self.create_manifest(root)
        jnajar = py.path.local(__file__).dirpath('jvm', 'src', 'jna.jar')
        classlist = self.create_classlist(root, [jnajar])
        jarfile = py.path.local(basename + '.jar')
        self.log.info('Creating jar file')
        oldpath = root.chdir()
        subprocess.call(['jar', 'cmf', str(manifest), str(jarfile), '@'+str(classlist)])
        oldpath.chdir()

        # create a convenience script
        newexename = basename
        f = file(newexename, 'w')
        f.write("""#!/bin/bash
LEDIT=`type -p ledit`
EXE=`readlink $0`
if [ -z $EXE ]
then
    EXE=$0
fi
$LEDIT java -Xmx256m -jar $EXE.jar "$@"
""")
        f.close()
        os.chmod(newexename, 0755)

    def create_manifest(self, root):
        filename = root.join('manifest.txt')
        manifest = filename.open('w')
        manifest.write('Main-class: pypy.Main\n\n')
        manifest.close()
        return filename

    def create_classlist(self, root, additional_jars=[]):
        from py.compat import subprocess
        # first, uncompress additional jars
        for jarfile in additional_jars:
            oldpwd = root.chdir()
            subprocess.call(['jar', 'xf', str(jarfile)])
            oldpwd.chdir()
        filename = root.join('classlist.txt')
        classlist = filename.open('w')
        classfiles = list(root.visit('*.class', True))
        classfiles += root.visit('*.so', True)
        classfiles += root.visit('*.dll', True)
        classfiles += root.visit('*.jnilib', True)
        for classfile in classfiles:
            print >> classlist, classfile.relto(root)
        classlist.close()
        return filename

    def task_run_jvm(self):
        pass
    task_run_jvm = taskdef(task_run_jvm, ['compile_jvm'],
                           'XXX')

    def proceed(self, goals):
        if not goals:
            if self.default_goal:
                goals = [self.default_goal]
            else:
                self.log.info("nothing to do")
                return
        elif isinstance(goals, str):
            goals = [goals]
        goals.extend(self.extra_goals)
        goals = self.backend_select_goals(goals)
        return self._execute(goals, task_skip = self._maybe_skip())

    def from_targetspec(targetspec_dic, config=None, args=None,
                        empty_translator=None,
                        disable=[],
                        default_goal=None):
        if args is None:
            args = []

        driver = TranslationDriver(config=config, default_goal=default_goal,
                                   disable=disable)
        # patch some attributes of the os module to make sure they
        # have the same value on every platform.
        backend, ts = driver.get_backend_and_type_system()
        if backend in ('cli', 'jvm'):
            from pypy.translator.oosupport.support import patch_os
            driver.old_cli_defs = patch_os()
        
        target = targetspec_dic['target']
        spec = target(driver, args)

        try:
            entry_point, inputtypes, policy = spec
        except ValueError:
            entry_point, inputtypes = spec
            policy = None

        driver.setup(entry_point, inputtypes, 
                     policy=policy, 
                     extra=targetspec_dic,
                     empty_translator=empty_translator)

        return driver

    from_targetspec = staticmethod(from_targetspec)

    def prereq_checkpt_rtype(self):
        assert 'pypy.rpython.rmodel' not in sys.modules, (
            "cannot fork because the rtyper has already been imported")
    prereq_checkpt_rtype_lltype = prereq_checkpt_rtype
    prereq_checkpt_rtype_ootype = prereq_checkpt_rtype    

    # checkpointing support
    def _event(self, kind, goal, func):
        if kind == 'planned' and func.task_earlycheck:
            func.task_earlycheck(self)
        if kind == 'pre':
            fork_before = self.config.translation.fork_before
            if fork_before:
                fork_before, = self.backend_select_goals([fork_before])
                if not fork_before in self.done and fork_before == goal:
                    prereq = getattr(self, 'prereq_checkpt_%s' % goal, None)
                    if prereq:
                        prereq()
                    from pypy.translator.goal import unixcheckpoint
                    unixcheckpoint.restartable_point(auto='run')