Esempio n. 1
0
def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None):
    from Cython.Compiler.Main import compile, default_options
    from Cython.Compiler.Errors import CompileError, PyrexError

    if fingerprint:
        if not os.path.exists(options.cache):
            try:
                os.mkdir(options.cache)
            except:
                if not os.path.exists(options.cache):
                    raise
        # Cython-generated c files are highly compressible.
        # (E.g. a compression ratio of about 10 for Sage).
        fingerprint_file = os.path.join(
            options.cache, fingerprint + '-' + os.path.basename(c_file) + '.gz')
        if os.path.exists(fingerprint_file):
            if not quiet:
                print("Found compiled %s in cache" % pyx_file)
            os.utime(fingerprint_file, None)
            open(c_file, 'wb').write(gzip.open(fingerprint_file).read())
            return
    if not quiet:
        print("Cythonizing %s" % pyx_file)
    if options is None:
        options = CompilationOptions(default_options)
    options.output_file = c_file

    any_failures = 0
    try:
        result = compile([pyx_file], options)
        if result.num_errors > 0:
            any_failures = 1
    except (EnvironmentError, PyrexError), e:
        sys.stderr.write('%s\n' % e)
        any_failures = 1
Esempio n. 2
0
def cythonize(source, includes=(),
              output_h=os.curdir):
    name, ext = os.path.splitext(source)
    output_c = name + '.c'
    #
    from Cython.Compiler.Main import \
         CompilationOptions, default_options, \
         compile, \
         PyrexError
    #
    options = CompilationOptions(default_options)
    options.output_file = output_c
    options.include_path = includes
    #
    from Cython.Compiler import Options
    Options.generate_cleanup_code = 3
    #
    any_failures = 0
    try:
        result = compile(source, options)
        if result.num_errors > 0:
            any_failures = 1
    except (EnvironmentError, PyrexError), e:
        sys.stderr.write(str(e) + '\n')
        any_failures = 1
Esempio n. 3
0
def cythonize(source,
              includes=(),
              destdir_c=None,
              destdir_h=None,
              wdir=None):
    from Cython.Compiler.Main import \
         CompilationOptions, default_options, \
         compile, \
         PyrexError
    from Cython.Compiler import Options
    cwd = os.getcwd()
    try:
        name, ext = os.path.splitext(source)
        outputs_c = [name+'.c']
        outputs_h = [name+'.h', name+'_api.h']
        # change working directory
        if wdir:
            os.chdir(wdir)
        # run Cython on source
        options = CompilationOptions(default_options)
        options.output_file = outputs_c[0]
        options.include_path = list(includes)
        Options.generate_cleanup_code = 3
        any_failures = 0
        try:
            result = compile(source, options)
            if result.num_errors > 0:
                any_failures = 1
        except (EnvironmentError, PyrexError):
            e = sys.exc_info()[1]
            sys.stderr.write(str(e) + '\n')
            any_failures = 1
        if any_failures:
            for output in outputs_c + outputs_h:
                try:
                    os.remove(output)
                except OSError:
                    pass
            return 1
        # move ouputs
        for destdir, outputs in (
            (destdir_c, outputs_c),
            (destdir_h, outputs_h)):
            if destdir is None: continue
            for output in outputs:
                dest = os.path.join(
                    destdir, os.path.basename(output))
                try:
                    os.remove(dest)
                except OSError:
                    pass
                os.rename(output, dest)
        #
        return 0
    #
    finally:
        os.chdir(cwd)
Esempio n. 4
0
def run_cython(cython_file, c_file):
    assert have_cython
    msg('Cythonizing %s -> %s', cython_file, c_file)
    options = CythonCompilationOptions(cython_default_options)
    options.output_file = c_file
    try:
        result = cython_compile([cython_file], options)
    except (EnvironmentError, PyrexError), e:
        error_msg(str(e))
Esempio n. 5
0
    def test_func_transform(self):
        options = CompilationOptions(default_options, ctypes=True)
        context = options.create_context()
        t = self.run_pipeline([NormalizeTree(self), ExternDefTransform(context)],
u"""\
cdef extern from "stdio.h":
    int printf(char *, int *)\
""")
        print self.codeToString(t)
        self.assertEquals(self.codeToString(t),
"""\
with (cython.ctypes_extern)('stdio.h'):
    printf = (cython.ctypes_func)('printf','c',ctypes.c_int,ctypes.POINTER(ctypes.c_char),ctypes.POINTER(ctypes.c_int))\
""")
Esempio n. 6
0
def cythonize_one(pyx_file, c_file, options=None):
    from Cython.Compiler.Main import compile, default_options
    from Cython.Compiler.Errors import CompileError, PyrexError

    if options is None:
        options = CompilationOptions(default_options)
    options.output_file = c_file

    any_failures = 0
    try:
        result = compile([pyx_file], options)
        if result.num_errors > 0:
            any_failures = 1
    except (EnvironmentError, PyrexError), e:
        sys.stderr.write(str(e) + '\n')
        any_failures = 1
Esempio n. 7
0
def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True):
    from Cython.Compiler.Main import compile, default_options
    from Cython.Compiler.Errors import CompileError, PyrexError

    if fingerprint:
        if not os.path.exists(options.cache):
            try:
                os.mkdir(options.cache)
            except:
                if not os.path.exists(options.cache):
                    raise
        # Cython-generated c files are highly compressible.
        # (E.g. a compression ratio of about 10 for Sage).
        fingerprint_file = join_path(
            options.cache, "%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext))
        if os.path.exists(fingerprint_file):
            if not quiet:
                print("Found compiled %s in cache" % pyx_file)
            os.utime(fingerprint_file, None)
            g = gzip_open(fingerprint_file, 'rb')
            try:
                f = open(c_file, 'wb')
                try:
                    shutil.copyfileobj(g, f)
                finally:
                    f.close()
            finally:
                g.close()
            return
    if not quiet:
        print("Cythonizing %s" % pyx_file)
    if options is None:
        options = CompilationOptions(default_options)
    options.output_file = c_file

    any_failures = 0
    try:
        result = compile([pyx_file], options)
        if result.num_errors > 0:
            any_failures = 1
    except (EnvironmentError, PyrexError), e:
        sys.stderr.write('%s\n' % e)
        any_failures = 1
        # XXX
        import traceback
        traceback.print_exc()
Esempio n. 8
0
    def run(self):
        """Run - pass execution to generate_qcd"""

        filepaths = generate_qcd(self.num_colours, self.precision,
                                 self.representation)

        from Cython.Build.Dependencies import cythonize_one
        from Cython.Compiler.Main import CompilationOptions

        options = CompilationOptions()
        options.cplus = True

        for src_path in filepaths:
            if not src_path.endswith(".pyx"):
                raise ValueError("Aborted attempt to cythonize file '{}'"
                                 .format(src_path))

            dest_path = src_path[-3:] + "cpp"
            cythonize_one(src_path, dest_path, None, False, options=options)
Esempio n. 9
0
def extract(path, **kwargs):

    name = os.path.splitext(os.path.relpath(path))[0].replace('/', '.')

    options = CompilationOptions()
    options.include_path.append('include')
    options.language_level = 2
    options.compiler_directives = dict(
        c_string_type='str',
        c_string_encoding='ascii',
    )

    context = options.create_context()

    tree = parse_from_strings(name, open(path).read().decode('utf8'), context, **kwargs)

    extractor = Visitor({'file': path})
    extractor.visit(tree)
    return extractor.events
Esempio n. 10
0
def cythonize():
   try:
      from Cython.Compiler.Main import CompilationOptions, default_options, \
         compile, PyrexError
      from Cython.Compiler import Options
      import subprocess 

      for code in cythoncodes:
         source = code[0] + '.pyx'
         options = CompilationOptions(default_options)
         options.output_file = code[0] + '.c'
         options.include_path = code[1]
         Options.generate_cleanup_code = 3
         any_failures = False
         try:
            result = compile(source, options)
            if result.num_errors > 0: 
               any_failures = True
            if not any_failures:
               callist = [cythcompiler,'-shared','-fwrapv','-Wall','-fno-strict-aliasing']
               for x in CFLAGS:
                  callist.append(x)
               for x in code[1]:
                  callist.append('-L' + x)
               callist.append('-o')
               callist.append('_' + code[0] + '.so')
               callist.append(code[0] + '.c')
               subprocess.call(callist)
         except (EnvironmentError, PyrexError):
            e = sys.exc_info()[1]
            sys.stderr.write(str(e) + '\n')
            any_failures = True
         if any_failures:
            try:  os.remove(code[0] + '.c')
            except OSError: pass
   except:
      raise ValueError
Esempio n. 11
0
    def cython_sources(self, sources, extension):
        """
        Walk the list of source files in 'sources', looking for Cython
        source files (.pyx and .py).  Run Cython on all that are
        found, and return a modified 'sources' list with Cython source
        files replaced by the generated C (or C++) files.
        """
        try:
            from Cython.Compiler.Main \
                import CompilationOptions, \
                       default_options as pyrex_default_options, \
                       compile as cython_compile
            from Cython.Compiler.Errors import PyrexError
        except ImportError:
            e = sys.exc_info()[1]
            print("failed to import Cython: %s" % e)
            raise DistutilsPlatformError(
                "Cython does not appear to be installed")

        new_sources = []
        pyrex_sources = []
        pyrex_targets = {}

        # Setup create_list and cplus from the extension options if
        # Cython.Distutils.extension.Extension is used, otherwise just
        # use what was parsed from the command-line or the configuration file.
        # cplus will also be set to true is extension.language is equal to
        # 'C++' or 'c++'.
        #try:
        #    create_listing = self.pyrex_create_listing or \
        #                        extension.pyrex_create_listing
        #    cplus = self.pyrex_cplus or \
        #                extension.pyrex_cplus or \
        #                (extension.language != None and \
        #                    extension.language.lower() == 'c++')
        #except AttributeError:
        #    create_listing = self.pyrex_create_listing
        #    cplus = self.pyrex_cplus or \
        #                (extension.language != None and \
        #                    extension.language.lower() == 'c++')

        create_listing = self.pyrex_create_listing or \
            getattr(extension, 'pyrex_create_listing', 0)
        line_directives = self.pyrex_line_directives or \
            getattr(extension, 'pyrex_line_directives', 0)
        no_c_in_traceback = self.no_c_in_traceback or \
            getattr(extension, 'no_c_in_traceback', 0)
        cplus = self.pyrex_cplus or getattr(extension, 'pyrex_cplus', 0) or \
                (extension.language and extension.language.lower() == 'c++')
        pyrex_gen_pxi = self.pyrex_gen_pxi or getattr(extension,
                                                      'pyrex_gen_pxi', 0)
        pyrex_gdb = self.pyrex_gdb or getattr(extension, 'pyrex_gdb', False)
        # Set up the include_path for the Cython compiler:
        #    1.    Start with the command line option.
        #    2.    Add in any (unique) paths from the extension
        #        pyrex_include_dirs (if Cython.Distutils.extension is used).
        #    3.    Add in any (unique) paths from the extension include_dirs
        includes = self.pyrex_include_dirs
        try:
            for i in extension.pyrex_include_dirs:
                if not i in includes:
                    includes.append(i)
        except AttributeError:
            pass
        for i in extension.include_dirs:
            if not i in includes:
                includes.append(i)

        # Set up Cython compiler directives:
        #    1. Start with the command line option.
        #    2. Add in any (unique) entries from the extension
        #         pyrex_directives (if Cython.Distutils.extension is used).
        directives = self.pyrex_directives
        if hasattr(extension, "pyrex_directives"):
            directives.update(extension.pyrex_directives)

        # Set the target_ext to '.c'.  Cython will change this to '.cpp' if
        # needed.
        if cplus:
            target_ext = '.cpp'
        else:
            target_ext = '.c'

        # Decide whether to drop the generated C files into the temp dir
        # or the source tree.

        if not self.inplace and (self.pyrex_c_in_temp
                                 or getattr(extension, 'pyrex_c_in_temp', 0)):
            target_dir = os.path.join(self.build_temp, "pyrex")
            for package_name in extension.name.split('.')[:-1]:
                target_dir = os.path.join(target_dir, package_name)
        else:
            target_dir = None

        newest_dependency = None
        for source in sources:
            (base, ext) = os.path.splitext(os.path.basename(source))
            if ext == ".py":
                # FIXME: we might want to special case this some more
                ext = '.pyx'
            if ext == ".pyx":  # Cython source file
                output_dir = target_dir or os.path.dirname(source)
                new_sources.append(os.path.join(output_dir, base + target_ext))
                pyrex_sources.append(source)
                pyrex_targets[source] = new_sources[-1]
            elif ext == '.pxi' or ext == '.pxd':
                if newest_dependency is None \
                        or newer(source, newest_dependency):
                    newest_dependency = source
            else:
                new_sources.append(source)

        if not pyrex_sources:
            return new_sources

        module_name = extension.name

        for source in pyrex_sources:
            target = pyrex_targets[source]
            depends = [source] + list(extension.depends or ())
            rebuild = self.force or newer_group(depends, target, 'newer')
            if not rebuild and newest_dependency is not None:
                rebuild = newer(newest_dependency, target)
            if rebuild:
                log.info("cythoning %s to %s", source, target)
                self.mkpath(os.path.dirname(target))
                if self.inplace:
                    output_dir = os.curdir
                else:
                    output_dir = self.build_lib
                options = CompilationOptions(
                    pyrex_default_options,
                    use_listing_file=create_listing,
                    include_path=includes,
                    compiler_directives=directives,
                    output_file=target,
                    cplus=cplus,
                    emit_linenums=line_directives,
                    c_line_in_traceback=not no_c_in_traceback,
                    generate_pxi=pyrex_gen_pxi,
                    output_dir=output_dir,
                    gdb_debug=pyrex_gdb)
                result = cython_compile(source,
                                        options=options,
                                        full_module_name=module_name)
            else:
                log.info("skipping '%s' Cython extension (up-to-date)", target)

        return new_sources
Esempio n. 12
0
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
              exclude_failures=False, **options):
    """
    Compile a set of source modules into C/C++ files and return a list of distutils
    Extension objects for them.

    As module list, pass either a glob pattern, a list of glob patterns or a list of
    Extension objects.  The latter allows you to configure the extensions separately
    through the normal distutils options.

    When using glob patterns, you can exclude certain module names explicitly
    by passing them into the 'exclude' option.

    For parallel compilation, set the 'nthreads' option to the number of
    concurrent builds.

    For a broad 'try to compile' mode that ignores compilation failures and
    simply excludes the failed extensions, pass 'exclude_failures=True'. Note
    that this only really makes sense for compiling .py files which can also
    be used without compilation.

    Additional compilation options can be passed as keyword arguments.
    """
    if 'include_path' not in options:
        options['include_path'] = ['.']
    if 'common_utility_include_dir' in options:
        if options.get('cache'):
            raise NotImplementedError("common_utility_include_dir does not yet work with caching")
        if not os.path.exists(options['common_utility_include_dir']):
            os.makedirs(options['common_utility_include_dir'])
    c_options = CompilationOptions(**options)
    cpp_options = CompilationOptions(**options); cpp_options.cplus = True
    ctx = c_options.create_context()
    options = c_options
    module_list = create_extension_list(
        module_list,
        exclude=exclude,
        ctx=ctx,
        quiet=quiet,
        exclude_failures=exclude_failures,
        aliases=aliases)
    deps = create_dependency_tree(ctx, quiet=quiet)
    build_dir = getattr(options, 'build_dir', None)

    modules_by_cfile = {}
    to_compile = []
    for m in module_list:
        if build_dir:
            root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
            def copy_to_build_dir(filepath, root=root):
                filepath_abs = os.path.realpath(os.path.abspath(filepath))
                if os.path.isabs(filepath):
                    filepath = filepath_abs
                if filepath_abs.startswith(root):
                    mod_dir = os.path.join(build_dir,
                            os.path.dirname(_relpath(filepath, root)))
                    if not os.path.isdir(mod_dir):
                        os.makedirs(mod_dir)
                    shutil.copy(filepath, mod_dir)
            for dep in m.depends:
                copy_to_build_dir(dep)

        new_sources = []
        for source in m.sources:
            base, ext = os.path.splitext(source)
            if ext in ('.pyx', '.py'):
                if m.language == 'c++':
                    c_file = base + '.cpp'
                    options = cpp_options
                else:
                    c_file = base + '.c'
                    options = c_options

                # setup for out of place build directory if enabled
                if build_dir:
                    c_file = os.path.join(build_dir, c_file)
                    dir = os.path.dirname(c_file)
                    if not os.path.isdir(dir):
                        os.makedirs(dir)

                if os.path.exists(c_file):
                    c_timestamp = os.path.getmtime(c_file)
                else:
                    c_timestamp = -1

                # Priority goes first to modified files, second to direct
                # dependents, and finally to indirect dependents.
                if c_timestamp < deps.timestamp(source):
                    dep_timestamp, dep = deps.timestamp(source), source
                    priority = 0
                else:
                    dep_timestamp, dep = deps.newest_dependency(source)
                    priority = 2 - (dep in deps.immediate_dependencies(source))
                if force or c_timestamp < dep_timestamp:
                    if not quiet:
                        if source == dep:
                            print("Compiling %s because it changed." % source)
                        else:
                            print("Compiling %s because it depends on %s." % (source, dep))
                    if not force and hasattr(options, 'cache'):
                        extra = m.language
                        fingerprint = deps.transitive_fingerprint(source, extra)
                    else:
                        fingerprint = None
                    to_compile.append((priority, source, c_file, fingerprint, quiet,
                                       options, not exclude_failures))
                new_sources.append(c_file)
                if c_file not in modules_by_cfile:
                    modules_by_cfile[c_file] = [m]
                else:
                    modules_by_cfile[c_file].append(m)
            else:
                new_sources.append(source)
                if build_dir:
                    copy_to_build_dir(source)
        m.sources = new_sources

    if hasattr(options, 'cache'):
        if not os.path.exists(options.cache):
            os.makedirs(options.cache)
    to_compile.sort()
    if nthreads:
        # Requires multiprocessing (or Python >= 2.6)
        try:
            import multiprocessing
            pool = multiprocessing.Pool(nthreads)
        except (ImportError, OSError):
            print("multiprocessing required for parallel cythonization")
            nthreads = 0
        else:
            try:
                pool.map(cythonize_one_helper, to_compile)
            finally:
                pool.close()
    if not nthreads:
        for args in to_compile:
            cythonize_one(*args[1:])

    if exclude_failures:
        failed_modules = set()
        for c_file, modules in modules_by_cfile.iteritems():
            if not os.path.exists(c_file):
                failed_modules.update(modules)
            elif os.path.getsize(c_file) < 200:
                f = io_open(c_file, 'r', encoding='iso8859-1')
                try:
                    if f.read(len('#error ')) == '#error ':
                        # dead compilation result
                        failed_modules.update(modules)
                finally:
                    f.close()
        if failed_modules:
            for module in failed_modules:
                module_list.remove(module)
            print("Failed compilations: %s" % ', '.join(sorted([
                module.name for module in failed_modules])))

    if hasattr(options, 'cache'):
        cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
    # cythonize() is often followed by the (non-Python-buffered)
    # compiler output, flush now to avoid interleaving output.
    sys.stdout.flush()
    return module_list
Esempio n. 13
0
def parse_command_line(args):

    from Cython.Compiler.Main import \
        CompilationOptions, default_options

    def pop_arg():
        if args:
            return args.pop(0)
        else:
            bad_usage()

    def get_param(option):
        tail = option[2:]
        if tail:
            return tail
        else:
            return pop_arg()

    options = CompilationOptions(default_options)
    sources = []
    while args:
        if args[0].startswith("-"):
            option = pop_arg()
            if option in ("-V", "--version"):
                options.show_version = 1
            elif option in ("-l", "--create-listing"):
                options.use_listing_file = 1
            elif option in ("-+", "--cplus"):
                options.cplus = 1
            elif option.startswith("--embed"):
                ix = option.find('=')
                if ix == -1:
                    Options.embed = "main"
                else:
                    Options.embed = option[ix+1:]
            elif option.startswith("-I"):
                options.include_path.append(get_param(option))
            elif option == "--include-dir":
                options.include_path.append(pop_arg())
            elif option in ("-w", "--working"):
                options.working_path = pop_arg()
            elif option in ("-o", "--output-file"):
                options.output_file = pop_arg()
            elif option in ("-r", "--recursive"):
                options.recursive = 1
            elif option in ("-t", "--timestamps"):
                options.timestamps = 1
            elif option in ("-f", "--force"):
                options.timestamps = 0
            elif option in ("-v", "--verbose"):
                options.verbose += 1
            elif option in ("-p", "--embed-positions"):
                Options.embed_pos_in_docstring = 1
            elif option in ("-z", "--pre-import"):
                Options.pre_import = pop_arg()
            elif option == "--cleanup":
                Options.generate_cleanup_code = int(pop_arg())
            elif option in ("-D", "--no-docstrings"):
                Options.docstrings = False
            elif option in ("-a", "--annotate"):
                Options.annotate = True
            elif option == "--convert-range":
                Options.convert_range = True
            elif option == "--line-directives":
                options.emit_linenums = True
            elif option == "--no-c-in-traceback":
                options.c_line_in_traceback = False
            elif option == "--gdb":
                options.gdb_debug = True
                options.output_dir = os.curdir
            elif option == '-2':
                options.language_level = 2
            elif option == '-3':
                options.language_level = 3
            elif option == "--fast-fail":
                Options.fast_fail = True
            elif option in ('-Werror', '--warning-errors'):
                Options.warning_errors = True
            elif option == "--disable-function-redefinition":
                Options.disable_function_redefinition = True
            elif option == "--directive" or option.startswith('-X'):
                if option.startswith('-X') and option[2:].strip():
                    x_args = option[2:]
                else:
                    x_args = pop_arg()
                try:
                    options.compiler_directives = Options.parse_directive_list(
                        x_args, relaxed_bool=True,
                        current_settings=options.compiler_directives)
                except ValueError, e:
                    sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
                    sys.exit(1)
            elif option.startswith('--debug'):
                option = option[2:].replace('-', '_')
                import DebugFlags
                if option in dir(DebugFlags):
                    setattr(DebugFlags, option, True)
                else:
                    sys.stderr.write("Unknown debug flag: %s\n" % option)
                    bad_usage()
            elif option in ('-h', '--help'):
                sys.stdout.write(usage)
                sys.exit(0)
            else:
                sys.stderr.write("Unknown compiler flag: %s\n" % option)
                sys.exit(1)
        else:
            arg = pop_arg()
            if arg.endswith(".pyx"):
                sources.append(arg)
            elif arg.endswith(".py"):
                # maybe do some other stuff, but this should work for now
                sources.append(arg)
            else:
                sys.stderr.write(
                    "cython: %s: Unknown filename suffix\n" % arg)
Esempio n. 14
0
def parse_command_line(args):

    from Cython.Compiler.Main import \
        CompilationOptions, default_options

    def pop_arg():
        if args:
            return args.pop(0)
        else:
            bad_usage()
    
    def get_param(option):
        tail = option[2:]
        if tail:
            return tail
        else:
            return pop_arg()

    options = CompilationOptions(default_options)
    sources = []
    while args:
        if args[0].startswith("-"):
            option = pop_arg()
            if option in ("-V", "--version"):
                options.show_version = 1
            elif option in ("-l", "--create-listing"):
                options.use_listing_file = 1
            elif option in ("-C", "--compile"):
                options.c_only = 0
            elif option in ("-X", "--link"):
                options.c_only = 0
                options.obj_only = 0
            elif option in ("-+", "--cplus"):
                options.cplus = 1
            elif option.startswith("-I"):
                options.include_path.append(get_param(option))
            elif option == "--include-dir":
                options.include_path.append(pop_arg())
            elif option in ("-w", "--working"):
                options.working_path = pop_arg()
            elif option in ("-o", "--output-file"):
                options.output_file = pop_arg()
            elif option in ("-r", "--recursive"):
                options.recursive = 1
            elif option in ("-t", "--timestamps"):
                options.timestamps = 1
            elif option in ("-f", "--force"):
                options.timestamps = 0
            elif option in ("-v", "--verbose"):
                options.verbose += 1
            elif option in ("-p", "--embed-positions"):
                Options.embed_pos_in_docstring = 1
            elif option in ("-z", "--pre-import"):
                Options.pre_import = pop_arg()
            elif option == "--incref-local-binop":
                Options.incref_local_binop = 1
            elif option == "--cleanup":
                Options.generate_cleanup_code = int(pop_arg())
            elif option in ("-D", "--no-docstrings"):
                Options.docstrings = False
            elif option in ("-a", "--annotate"):
                Options.annotate = True
            elif option == "--convert-range":
                Options.convert_range = True
            elif option == "--line-directives":
                options.emit_linenums = True
            elif option in ("-X", "--directive"):
                try:
                    options.pragma_overrides = Options.parse_option_list(pop_arg())
                except ValueError, e:
                    sys.stderr.write("Error in compiler directive: %s\n" % e.message)
                    sys.exit(1)
            else:
                bad_usage()
        else:
            arg = pop_arg()
            if arg.endswith(".pyx"):
                sources.append(arg)
            elif arg.endswith(".py"):
                # maybe do some other stuff, but this should work for now
                sources.append(arg)
            elif arg.endswith(".o"):
                options.objects.append(arg)
            else:
                sys.stderr.write(
                    "cython: %s: Unknown filename suffix\n" % arg)
Esempio n. 15
0
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **options):
    if 'include_path' not in options:
        options['include_path'] = ['.']
    c_options = CompilationOptions(**options)
    cpp_options = CompilationOptions(**options); cpp_options.cplus = True
    ctx = c_options.create_context()
    module_list = create_extension_list(
        module_list,
        exclude=exclude,
        ctx=ctx,
        aliases=aliases)
    deps = create_dependency_tree(ctx)
    to_compile = []
    for m in module_list:
        new_sources = []
        for source in m.sources:
            base, ext = os.path.splitext(source)
            if ext in ('.pyx', '.py'):
                if m.language == 'c++':
                    c_file = base + '.cpp'
                    options = cpp_options
                else:
                    c_file = base + '.c'
                    options = c_options
                if os.path.exists(c_file):
                    c_timestamp = os.path.getmtime(c_file)
                else:
                    c_timestamp = -1
                # Priority goes first to modified files, second to direct
                # dependents, and finally to indirect dependents.
                if c_timestamp < deps.timestamp(source):
                    dep_timestamp, dep = deps.timestamp(source), source
                    priority = 0
                else:
                    dep_timestamp, dep = deps.newest_dependency(source)
                    priority = 2 - (dep in deps.immediate_dependencies(source))
                if c_timestamp < dep_timestamp:
                    if not quiet:
                        if source == dep:
                            print("Compiling %s because it changed." % source)
                        else:
                            print("Compiling %s because it depends on %s." % (source, dep))
                    to_compile.append((priority, source, c_file, options))
                new_sources.append(c_file)
            else:
                new_sources.append(source)
        m.sources = new_sources
    to_compile.sort()
    if nthreads:
        # Requires multiprocessing (or Python >= 2.6)
        try:
            import multiprocessing
            pool = multiprocessing.Pool(nthreads)
            pool.map(cythonize_one_helper, to_compile)
        except ImportError:
            print("multiprocessing required for parallel cythonization")
            nthreads = 0
    if not nthreads:
        for priority, pyx_file, c_file, options in to_compile:
            cythonize_one(pyx_file, c_file, options)
    return module_list
Esempio n. 16
0
def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
              exclude_failures=False, **options):
    """
    Compile a set of source modules into C/C++ files and return a list of distutils
    Extension objects for them.

    As module list, pass either a glob pattern, a list of glob patterns or a list of
    Extension objects.  The latter allows you to configure the extensions separately
    through the normal distutils options.

    When using glob patterns, you can exclude certain module names explicitly
    by passing them into the 'exclude' option.

    For parallel compilation, set the 'nthreads' option to the number of
    concurrent builds.

    For a broad 'try to compile' mode that ignores compilation failures and
    simply excludes the failed extensions, pass 'exclude_failures=True'. Note
    that this only really makes sense for compiling .py files which can also
    be used without compilation.

    Additional compilation options can be passed as keyword arguments.
    """
    if 'include_path' not in options:
        options['include_path'] = ['.']
    c_options = CompilationOptions(**options)
    cpp_options = CompilationOptions(**options); cpp_options.cplus = True
    ctx = c_options.create_context()
    module_list = create_extension_list(
        module_list,
        exclude=exclude,
        ctx=ctx,
        quiet=quiet,
        exclude_failures=exclude_failures,
        aliases=aliases)
    deps = create_dependency_tree(ctx, quiet=quiet)
    modules_by_cfile = {}
    to_compile = []
    for m in module_list:
        new_sources = []
        for source in m.sources:
            base, ext = os.path.splitext(source)
            if ext in ('.pyx', '.py'):
                if m.language == 'c++':
                    c_file = base + '.cpp'
                    options = cpp_options
                else:
                    c_file = base + '.c'
                    options = c_options
                if os.path.exists(c_file):
                    c_timestamp = os.path.getmtime(c_file)
                else:
                    c_timestamp = -1
                    
                # Priority goes first to modified files, second to direct
                # dependents, and finally to indirect dependents.
                if c_timestamp < deps.timestamp(source):
                    dep_timestamp, dep = deps.timestamp(source), source
                    priority = 0
                else:
                    dep_timestamp, dep = deps.newest_dependency(source)
                    priority = 2 - (dep in deps.immediate_dependencies(source))
                if force or c_timestamp < dep_timestamp:
                    if not quiet:
                        if source == dep:
                            print("Compiling %s because it changed." % source)
                        else:
                            print("Compiling %s because it depends on %s." % (source, dep))
                    if not force and hasattr(options, 'cache'):
                        extra = m.language
                        fingerprint = deps.transitive_fingerprint(source, extra)
                    else:
                        fingerprint = None
                    to_compile.append((priority, source, c_file, fingerprint, quiet,
                                       options, not exclude_failures))
                new_sources.append(c_file)
                if c_file not in modules_by_cfile:
                    modules_by_cfile[c_file] = [m]
                else:
                    modules_by_cfile[c_file].append(m)
            else:
                new_sources.append(source)
        m.sources = new_sources
    if hasattr(options, 'cache'):
        if not os.path.exists(options.cache):
            os.mkdir(options.cache)
    to_compile.sort()
    if nthreads:
        # Requires multiprocessing (or Python >= 2.6)
        try:
            import multiprocessing
            pool = multiprocessing.Pool(nthreads)
            pool.map(cythonize_one_helper, to_compile)
        except ImportError:
            print("multiprocessing required for parallel cythonization")
            nthreads = 0
    if not nthreads:
        for args in to_compile:
            cythonize_one(*args[1:])
    if exclude_failures:
        failed_modules = set()
        for c_file, modules in modules_by_cfile.iteritems():
            if not os.path.exists(c_file):
                failed_modules.update(modules)
        for module in failed_modules:
            module_list.remove(module)
    if hasattr(options, 'cache'):
        cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
    return module_list
Esempio n. 17
0
File: parse.py Progetto: davesc/PyAV
from Cython.Compiler.Main import compile_single, CompilationOptions
from Cython.Compiler.TreeFragment import parse_from_strings
from Cython.Compiler.Visitor import TreeVisitor
from Cython.Compiler import Nodes
from Cython.Compiler.AutoDocTransforms import EmbedSignature

options = CompilationOptions()
options.include_path.append('include')
options.language_level = 2
options.compiler_directives = dict(
    c_string_type='str',
    c_string_encoding='ascii',
)

ctx = options.create_context()

tree = parse_from_strings('include.libavutil.avutil', open('scratchpad/test.pxd').read().decode('utf8'), ctx)

class Visitor(TreeVisitor):

    def __init__(self, state=None):
        super(Visitor, self).__init__()
        self.state = dict(state or {})
        self.events = []

    def record_event(self, node, **kw):
        state = self.state.copy()
        state.update(**kw)
        state['pos'] = node.pos
        state['end_pos'] = node.end_pos()
        self.events.append(state)
Esempio n. 18
0
#
# It can be run in the build process of pyopenms when manual changes to the pyx
# files need to be made (not recommended!).
#
# == taken from autowrap/Main.py run(), lower part

infile = "pyopenms/pyopenms.pyx"

import cPickle
persisted_data_path = "include_dir.bin"
autowrap_include_dirs = cPickle.load(open(persisted_data_path, "rb"))
# autowrap_include_dirs = ['/usr/local/lib/python2.7/dist-packages/autowrap/data_files/boost', '/usr/local/lib/python2.7/dist-packages/autowrap/data_files', '/path/to/pyOpenMS/pxds', 'from Map cimport Map as _Map']

from Cython.Compiler.Main import compile, CompilationOptions
from Cython.Compiler.Options import directive_defaults
directive_defaults["boundscheck"] = False
directive_defaults["wraparound"] = False

options = dict(
    include_path=autowrap_include_dirs,
    compiler_directives=directive_defaults,
    #output_dir=".",
    #gdb_debug=True,
    cplus=True)

print "Compiling with Cython the file", infile
print "Using include_path", autowrap_include_dirs
options = CompilationOptions(**options)
compile(infile, options=options)
print "Success!"
Esempio n. 19
0
def parse_command_line(args):

    from Cython.Compiler.Main import \
        CompilationOptions, default_options

    def pop_arg():
        if args:
            return args.pop(0)
        else:
            bad_usage()

    def get_param(option):
        tail = option[2:]
        if tail:
            return tail
        else:
            return pop_arg()

    options = CompilationOptions(default_options)
    sources = []
    while args:
        if args[0].startswith("-"):
            option = pop_arg()
            if option in ("-V", "--version"):
                options.show_version = 1
            elif option in ("-l", "--create-listing"):
                options.use_listing_file = 1
            elif option in ("-+", "--cplus"):
                options.cplus = 1
            elif option == "--embed":
                Options.embed = "main"
            elif option.startswith("--embed="):
                Options.embed = option[8:]
            elif option.startswith("-I"):
                options.include_path.append(get_param(option))
            elif option == "--include-dir":
                options.include_path.append(pop_arg())
            elif option in ("-w", "--working"):
                options.working_path = pop_arg()
            elif option in ("-o", "--output-file"):
                options.output_file = pop_arg()
            elif option in ("-t", "--timestamps"):
                options.timestamps = 1
            elif option in ("-f", "--force"):
                options.timestamps = 0
            elif option in ("-v", "--verbose"):
                options.verbose += 1
            elif option in ("-p", "--embed-positions"):
                Options.embed_pos_in_docstring = 1
            elif option in ("-z", "--pre-import"):
                Options.pre_import = pop_arg()
            elif option == "--cleanup":
                Options.generate_cleanup_code = int(pop_arg())
            elif option in ("-D", "--no-docstrings"):
                Options.docstrings = False
            elif option in ("-a", "--annotate"):
                Options.annotate = True
            elif option == "--convert-range":
                Options.convert_range = True
            elif option == "--line-directives":
                options.emit_linenums = True
            elif option == "--no-c-in-traceback":
                options.c_line_in_traceback = False
            elif option == "--gdb":
                options.gdb_debug = True
                options.output_dir = os.curdir
            elif option == "--gdb-outdir":
                options.gdb_debug = True
                options.output_dir = pop_arg()
            elif option == "--lenient":
                Options.error_on_unknown_names = False
                Options.error_on_uninitialized = False
            elif option == '-2':
                options.language_level = 2
            elif option == '-3':
                options.language_level = 3
            elif option == "--capi-reexport-cincludes":
                options.capi_reexport_cincludes = True
            elif option == "--fast-fail":
                Options.fast_fail = True
            elif option in ('-Werror', '--warning-errors'):
                Options.warning_errors = True
            elif option in ('-Wextra', '--warning-extra'):
                options.compiler_directives.update(Options.extra_warnings)
            elif option == "--old-style-globals":
                Options.old_style_globals = True
            elif option == "--directive" or option.startswith('-X'):
                if option.startswith('-X') and option[2:].strip():
                    x_args = option[2:]
                else:
                    x_args = pop_arg()
                try:
                    options.compiler_directives = Options.parse_directive_list(
                        x_args, relaxed_bool=True,
                        current_settings=options.compiler_directives)
                except ValueError as e:
                    sys.stderr.write("Error in compiler directive: %s\n" % e.args[0])
                    sys.exit(1)
            elif option.startswith('--debug'):
                option = option[2:].replace('-', '_')
                from . import DebugFlags
                if option in dir(DebugFlags):
                    setattr(DebugFlags, option, True)
                else:
                    sys.stderr.write("Unknown debug flag: %s\n" % option)
                    bad_usage()
            elif option in ('-h', '--help'):
                sys.stdout.write(usage)
                sys.exit(0)
            else:
                sys.stderr.write("Unknown compiler flag: %s\n" % option)
                sys.exit(1)
        else:
            sources.append(pop_arg())
    if options.use_listing_file and len(sources) > 1:
        sys.stderr.write(
            "cython: Only one source file allowed when using -o\n")
        sys.exit(1)
    if len(sources) == 0 and not options.show_version:
        bad_usage()
    if Options.embed and len(sources) > 1:
        sys.stderr.write(
            "cython: Only one source file allowed when using -embed\n")
        sys.exit(1)
    return options, sources