def main():
    filelist = common.parseFileList([LIBS], recursive=True)
    try:
        import pefile
        filepath = pefile.__file__[:-1]
        filelist.append(filepath)
    except ImportError:
        print('pefile not installed...')
    for filename in filelist:
        if filename.endswith('.py'):
            filename = str(filename)
            try:
                pyx_to_dll(filename, inplace=True)
                print(filename, 'successful!')
            except Exception as e:
                print('ERROR:', filename, 'failed')
            try:
                os.remove(filename[:-2] + 'c')
            except Exception as e:
                # TODO: log exception
                pass

    # Cleanup build dirs
    walk = os.walk(LIBS)
    for path in walk:
        path = path[0]
        if os.path.basename(path) == '_pyxbld' and os.path.isdir(path):
            shutil.rmtree(path)
Ejemplo n.º 2
0
def main():
    filelist = utils.parseFileList([LIBS], recursive=True)
    try:
        import pefile
        filepath = pefile.__file__[:-1]
        filelist.append(filepath)
    except ImportError:
        print('pefile not installed...')
    for filename in filelist:
        if filename.endswith('.py'):
            filename = str(filename)
            try:
                pyx_to_dll(filename, inplace=True)
                print(filename, 'successful!')
            except Exception as e:
                print('ERROR:', filename, 'failed')
                logging.exception(e)
            try:
                os.remove(filename[:-2] + 'c')
            except Exception as e:
                logging.exception(e)

    # Cleanup build dirs
    walk = os.walk(LIBS)
    for path in walk:
        path = path[0]
        if os.path.basename(path) == '_pyxbld' and os.path.isdir(path):
            shutil.rmtree(path)
Ejemplo n.º 3
0
def get_openmp_flags():
    """Returns any OpenMP related flags if OpenMP is avaiable on the system.
    """
    omp_compile_flags, omp_link_flags = _get_openmp_flags()

    from textwrap import dedent
    try:
        from Cython.Distutils import Extension
        from pyximport import pyxbuild
    except ImportError:
        print("Unable to import Cython, disabling OpenMP for now.")
        return [], [], False

    from distutils.errors import CompileError, LinkError
    import shutil
    import tempfile
    test_code = dedent("""
    from cython.parallel import parallel, prange, threadid
    cimport openmp
    def n_threads():
        with nogil, parallel():
            openmp.omp_get_num_threads()
    """)
    tmp_dir = tempfile.mkdtemp()
    fname = os.path.join(tmp_dir, 'check_omp.pyx')
    with open(fname, 'w') as fp:
        fp.write(test_code)
    extension = Extension(
        name='check_omp',
        sources=[fname],
        include_dirs=base_includes,
        extra_compile_args=omp_compile_flags,
        extra_link_args=omp_link_flags,
    )
    has_omp = True
    try:
        pyxbuild.pyx_to_dll(fname, extension, pyxbuild_dir=tmp_dir)
        print("-" * 70)
        print("Using OpenMP.")
        print("-" * 70)
    except CompileError:
        print("*" * 70)
        print("Unable to compile OpenMP code. Not using OpenMP.")
        print("*" * 70)
        has_omp = False
    except LinkError:
        print("*" * 70)
        print("Unable to link OpenMP code. Not using OpenMP.")
        print("*" * 70)
        has_omp = False
    finally:
        shutil.rmtree(tmp_dir)

    if has_omp:
        return omp_compile_flags, omp_link_flags, True
    else:
        return [], [], False
Ejemplo n.º 4
0
 def compiled(self):
     if self._compiled_mod is None:
         import imp
         import os
         import os.path
         from distutils.extension import Extension
         from pyximport.pyxbuild import pyx_to_dll
         import cutiepy
         if DEBUG:
             display_highlighted_source(str(self))
         with tempfile.NamedTemporaryFile(mode='w',prefix='cutiepy_tmp_',suffix='.pyx',dir='.') as f:
             f.write(str(self))
             f.flush()
             folder, filename = os.path.split(f.name)
             extname, _ = os.path.splitext(filename)
             self._extension = Extension(extname,
                           sources=[filename],
                           include_dirs=[os.path.join(os.path.dirname(cutiepy.__file__), 'include')],
                           libraries=["sundials_cvode", "sundials_nvecserial"],
                           extra_compile_args=["-O3"]
                         )
             try:
                 module_path = pyx_to_dll(filename, ext=self._extension)
             except Exception as e:
                 raise e
             finally:
                 os.remove(os.path.join(folder, extname+'.c'))
         self._compiled_mod = imp.load_dynamic(filename.split('.')[0],module_path)
     self._compiled_mod.setup_generated_function(*self.predefined_constants_num)
     return self._compiled_mod
Ejemplo n.º 5
0
def build(t):
	"""Build a .pyx file"""
	from pyximport.pyxbuild import pyx_to_dll
	from pathlib import Path
	t = Path(pyx_to_dll(t))
	os.remove(t.parent.parent.parent / t.name)
	t.rename(t.parent.parent.parent / t.name)
Ejemplo n.º 6
0
    def build(self, force=False):
        """Build source into an extension module.  If force is False
        previously compiled module is returned.
        """
        if not self.shared_filesystem or self.rank == 0:
            with self._lock():
                if force or self.should_recompile():
                    self._message("Compiling code at:", self.src_path)
                    inc_dirs = [numpy.get_include()]
                    # Add pysph/base directory to inc_dirs for including spatial_hash.h
                    # for SpatialHashNNPS
                    inc_dirs.append(os.path.dirname(
                        os.path.realpath(__file__)))
                    extra_compile_args, extra_link_args = self._get_extra_args(
                    )

                    extension = Extension(
                        name=self.name,
                        sources=[self.src_path],
                        include_dirs=inc_dirs,
                        extra_compile_args=extra_compile_args,
                        extra_link_args=extra_link_args,
                        language="c++")

                    if not hasattr(sys.stdout, 'errors'):
                        # FIXME: This happens when nosetests replaces the
                        # stdout with the a Tee instance.  This Tee instance
                        # does not have errors which breaks the tests so we
                        # disable verbose reporting.
                        script_args = []
                    else:
                        script_args = ['--verbose']
                    try:
                        with CaptureMultipleStreams() as stream:
                            mod = pyxbuild.pyx_to_dll(
                                self.src_path,
                                extension,
                                pyxbuild_dir=self.build_dir,
                                force_rebuild=True,
                                setup_args={'script_args': script_args})
                    except (CompileError, LinkError):
                        hline = "*" * 80
                        print(hline + "\nERROR")
                        print(stream.get_output()[0])
                        print(stream.get_output()[1])
                        msg = "Compilation of code failed, please check "\
                                "error messages above."
                        print(hline + "\n" + msg)
                        sys.exit(1)
                    shutil.copy(mod, self.ext_path)
                else:
                    self._message("Precompiled code from:", self.src_path)
        if MPI is not None:
            self.comm.barrier()
Ejemplo n.º 7
0
 def _pyx_to_dll(self, filename, m, src):
     from pyximport.pyxbuild import pyx_to_dll
     pyxname = filename.new(ext='pyx')
     pyxfile = self.tmpdir.join(pyxname).ensure(file=True)
     pyxfile.write(src)
     if self.annotate:
         import Cython.Compiler.Options
         Cython.Compiler.Options.annotate = True
     dll = pyx_to_dll(str(pyxfile),
                      pyxbuild_dir=str(self.tmpdir),
                      setup_args=dict(include_dirs=self.include_dirs, ))
     if self.annotate and pyxfile.basename != 'annotate.pyx':
         htmlfile = pyxfile.new(ext='html')
         os.system('xdg-open %s' % htmlfile)
     return dll
Ejemplo n.º 8
0
 def compiled(self):
     if self._compiled_mod is None:
         import imp
         import os
         import os.path
         from distutils.extension import Extension
         from pyximport.pyxbuild import pyx_to_dll
         import cutiepy
         if DEBUG:
             display_highlighted_source(str(self))
         with tempfile.NamedTemporaryFile(mode='w',
                                          prefix='cutiepy_tmp_',
                                          suffix='.pyx',
                                          dir='.') as f:
             f.write(str(self))
             f.flush()
             folder, filename = os.path.split(f.name)
             extname, _ = os.path.splitext(filename)
             self._extension = Extension(
                 extname,
                 sources=[filename],
                 include_dirs=[
                     os.path.join(os.path.dirname(cutiepy.__file__),
                                  'include')
                 ],
                 libraries=["sundials_cvode", "sundials_nvecserial"],
                 extra_compile_args=["-O3"])
             try:
                 module_path = pyx_to_dll(filename, ext=self._extension)
             except Exception as e:
                 raise e
             finally:
                 os.remove(os.path.join(folder, extname + '.c'))
         self._compiled_mod = imp.load_dynamic(
             filename.split('.')[0], module_path)
     self._compiled_mod.setup_generated_function(
         *self.predefined_constants_num)
     return self._compiled_mod
Ejemplo n.º 9
0
def main():
    from optparse import OptionParser
    parser = OptionParser()
    parser.add_option("--no-cleanup", dest="cleanup_workdir",
                      action="store_false", default=True,
                      help="do not delete the generated C files (allows passing --no-cython on next run)")
    parser.add_option("--no-cleanup-sharedlibs", dest="cleanup_sharedlibs",
                      action="store_false", default=True,
                      help="do not delete the generated shared libary files (allows manual module experimentation)")
    parser.add_option("--no-cython", dest="with_cython",
                      action="store_false", default=True,
                      help="do not run the Cython compiler, only the C compiler")
    parser.add_option("--no-c", dest="use_c",
                      action="store_false", default=True,
                      help="do not test C compilation")
    parser.add_option("--no-cpp", dest="use_cpp",
                      action="store_false", default=True,
                      help="do not test C++ compilation")
    parser.add_option("--no-unit", dest="unittests",
                      action="store_false", default=True,
                      help="do not run the unit tests")
    parser.add_option("--no-doctest", dest="doctests",
                      action="store_false", default=True,
                      help="do not run the doctests")
    parser.add_option("--no-file", dest="filetests",
                      action="store_false", default=True,
                      help="do not run the file based tests")
    parser.add_option("--no-pyregr", dest="pyregr",
                      action="store_false", default=True,
                      help="do not run the regression tests of CPython in tests/pyregr/")    
    parser.add_option("--cython-only", dest="cython_only",
                      action="store_true", default=False,
                      help="only compile pyx to c, do not run C compiler or run the tests")
    parser.add_option("--no-refnanny", dest="with_refnanny",
                      action="store_false", default=True,
                      help="do not regression test reference counting")
    parser.add_option("--no-fork", dest="fork",
                      action="store_false", default=True,
                      help="do not fork to run tests")
    parser.add_option("--sys-pyregr", dest="system_pyregr",
                      action="store_true", default=False,
                      help="run the regression tests of the CPython installation")
    parser.add_option("-x", "--exclude", dest="exclude",
                      action="append", metavar="PATTERN",
                      help="exclude tests matching the PATTERN")
    parser.add_option("-C", "--coverage", dest="coverage",
                      action="store_true", default=False,
                      help="collect source coverage data for the Compiler")
    parser.add_option("--coverage-xml", dest="coverage_xml",
                      action="store_true", default=False,
                      help="collect source coverage data for the Compiler in XML format")
    parser.add_option("-A", "--annotate", dest="annotate_source",
                      action="store_true", default=True,
                      help="generate annotated HTML versions of the test source files")
    parser.add_option("--no-annotate", dest="annotate_source",
                      action="store_false",
                      help="do not generate annotated HTML versions of the test source files")
    parser.add_option("-v", "--verbose", dest="verbosity",
                      action="count", default=0,
                      help="display test progress, pass twice to print test names")
    parser.add_option("-T", "--ticket", dest="tickets",
                      action="append",
                      help="a bug ticket number to run the respective test in 'tests/*'")
    parser.add_option("-3", dest="language_level",
                      action="store_const", const=3, default=2,
                      help="set language level to Python 3 (useful for running the CPython regression tests)'")
    parser.add_option("--xml-output", dest="xml_output_dir", metavar="DIR",
                      help="write test results in XML to directory DIR")
    parser.add_option("--exit-ok", dest="exit_ok", default=False,
                      action="store_true",
                      help="exit without error code even on test failures")

    options, cmd_args = parser.parse_args()

    DISTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]))
    ROOTDIR = os.path.join(DISTDIR, 'tests')
    WORKDIR = os.path.join(os.getcwd(), 'BUILD')

    if sys.version_info[0] >= 3:
        options.doctests = False
        if options.with_cython:
            try:
                # try if Cython is installed in a Py3 version
                import Cython.Compiler.Main
            except Exception:
                # back out anything the import process loaded, then
                # 2to3 the Cython sources to make them re-importable
                cy_modules = [ name for name in sys.modules
                               if name == 'Cython' or name.startswith('Cython.') ]
                for name in cy_modules:
                    del sys.modules[name]
                # hasn't been refactored yet - do it now
                cy3_dir = os.path.join(WORKDIR, 'Cy3')
                if sys.version_info >= (3,1):
                    refactor_for_py3(DISTDIR, cy3_dir)
                elif os.path.isdir(cy3_dir):
                    sys.path.insert(0, cy3_dir)
                else:
                    options.with_cython = False

    WITH_CYTHON = options.with_cython

    if options.coverage or options.coverage_xml:
        if not WITH_CYTHON:
            options.coverage = options.coverage_xml = False
        else:
            from coverage import coverage as _coverage
            coverage = _coverage(branch=True)
            coverage.erase()
            coverage.start()

    if WITH_CYTHON:
        global CompilationOptions, pyrex_default_options, cython_compile
        from Cython.Compiler.Main import \
            CompilationOptions, \
            default_options as pyrex_default_options, \
            compile as cython_compile
        from Cython.Compiler import Errors
        Errors.LEVEL = 0 # show all warnings
        from Cython.Compiler import Options
        Options.generate_cleanup_code = 3   # complete cleanup code
        from Cython.Compiler import DebugFlags
        DebugFlags.debug_temp_code_comments = 1

    # RUN ALL TESTS!
    UNITTEST_MODULE = "Cython"
    UNITTEST_ROOT = os.path.join(os.getcwd(), UNITTEST_MODULE)
    if WITH_CYTHON:
        if os.path.exists(WORKDIR):
            for path in os.listdir(WORKDIR):
                if path in ("support", "Cy3"): continue
                shutil.rmtree(os.path.join(WORKDIR, path), ignore_errors=True)
    if not os.path.exists(WORKDIR):
        os.makedirs(WORKDIR)

    sys.stderr.write("Python %s\n" % sys.version)
    sys.stderr.write("\n")
    if WITH_CYTHON:
        from Cython.Compiler.Version import version
        sys.stderr.write("Running tests against Cython %s\n" % version)
    else:
        sys.stderr.write("Running tests without Cython.\n")

    if options.with_refnanny:
        from pyximport.pyxbuild import pyx_to_dll
        libpath = pyx_to_dll(os.path.join("Cython", "Runtime", "refnanny.pyx"),
                             build_in_temp=True,
                             pyxbuild_dir=os.path.join(WORKDIR, "support"))
        sys.path.insert(0, os.path.split(libpath)[0])
        CFLAGS.append("-DCYTHON_REFNANNY=1")

    if options.xml_output_dir and options.fork:
        # doesn't currently work together
        sys.stderr.write("Disabling forked testing to support XML test output\n")
        options.fork = False

    if WITH_CYTHON and options.language_level == 3:
        sys.stderr.write("Using Cython language level 3.\n")

    sys.stderr.write("\n")

    test_bugs = False
    if options.tickets:
        for ticket_number in options.tickets:
            test_bugs = True
            cmd_args.append('.*T%s$' % ticket_number)
    if not test_bugs:
        for selector in cmd_args:
            if selector.startswith('bugs'):
                test_bugs = True

    import re
    selectors = [ re.compile(r, re.I|re.U).search for r in cmd_args ]
    if not selectors:
        selectors = [ lambda x:True ]

    # Chech which external modules are not present and exclude tests
    # which depends on them (by prefix)

    missing_dep_excluder = MissingDependencyExcluder(EXT_DEP_MODULES) 
    version_dep_excluder = VersionDependencyExcluder(VER_DEP_MODULES) 
    exclude_selectors = [missing_dep_excluder, version_dep_excluder] # want to pring msg at exit

    if options.exclude:
        exclude_selectors += [ re.compile(r, re.I|re.U).search for r in options.exclude ]
    
    if not test_bugs:
        exclude_selectors += [ FileListExcluder("tests/bugs.txt") ]
    
    if sys.platform in ['win32', 'cygwin'] and sys.version_info < (2,6):
        exclude_selectors += [ lambda x: x == "run.specialfloat" ]

    languages = []
    if options.use_c:
        languages.append('c')
    if options.use_cpp:
        languages.append('cpp')

    test_suite = unittest.TestSuite()

    if options.unittests:
        collect_unittests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors)

    if options.doctests:
        collect_doctests(UNITTEST_ROOT, UNITTEST_MODULE + ".", test_suite, selectors)

    if options.filetests and languages:
        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors,
                                options.annotate_source, options.cleanup_workdir,
                                options.cleanup_sharedlibs, options.pyregr,
                                options.cython_only, languages, test_bugs,
                                options.fork, options.language_level)
        test_suite.addTest(filetests.build_suite())

    if options.system_pyregr and languages:
        filetests = TestBuilder(ROOTDIR, WORKDIR, selectors, exclude_selectors,
                                options.annotate_source, options.cleanup_workdir,
                                options.cleanup_sharedlibs, True,
                                options.cython_only, languages, test_bugs,
                                options.fork, options.language_level)
        test_suite.addTest(
            filetests.handle_directory(
                os.path.join(sys.prefix, 'lib', 'python'+sys.version[:3], 'test'),
                'pyregr'))

    if options.xml_output_dir:
        from Cython.Tests.xmlrunner import XMLTestRunner
        test_runner = XMLTestRunner(output=options.xml_output_dir,
                                    verbose=options.verbosity > 0)
    else:
        test_runner = unittest.TextTestRunner(verbosity=options.verbosity)

    result = test_runner.run(test_suite)

    if options.coverage or options.coverage_xml:
        coverage.stop()
        ignored_modules = ('Options', 'Version', 'DebugFlags', 'CmdLine')
        modules = [ module for name, module in sys.modules.items()
                    if module is not None and
                    name.startswith('Cython.Compiler.') and 
                    name[len('Cython.Compiler.'):] not in ignored_modules ]
        if options.coverage:
            coverage.report(modules, show_missing=0)
        if options.coverage_xml:
            coverage.xml_report(modules, outfile="coverage-report.xml")

    if missing_dep_excluder.tests_missing_deps:
        sys.stderr.write("Following tests excluded because of missing dependencies on your system:\n")
        for test in missing_dep_excluder.tests_missing_deps:
            sys.stderr.write("   %s\n" % test)

    if options.with_refnanny:
        import refnanny
        sys.stderr.write("\n".join([repr(x) for x in refnanny.reflog]))

    print("ALL DONE")

    if options.exit_ok:
        return_code = 0
    else:
        return_code = not result.wasSuccessful()

    try:
        check_thread_termination(ignore_seen=False)
        sys.exit(return_code)
    except PendingThreadsError:
        # normal program exit won't kill the threads, do it the hard way here
        os._exit(return_code)
Ejemplo n.º 10
0
def get_omp_flags():
    """Returns openmp flags if OpenMP is available.

    Implementation based on https://bitbucket.org/pysph/pysph

    """
    omp_compile_flags, omp_link_flags = ['-fopenmp'], ['-fopenmp']

    env_var = os.environ.get('USE_OPENMP', '')
    if env_var.lower() in ['0', 'false', 'n']:
        print("-" * 70)
        print("OpenMP disabled. Enable using 'USE_OPENMP'")
        print("-" * 70)
        return [], [], False

    from textwrap import dedent
    try:
        from Cython.Distutils import Extension
        from pyximport import pyxbuild
    except ImportError:
        print("Unable to import Cython, disabling OpenMP for now.")
        return [], [], False
    from distutils.errors import CompileError, LinkError
    import shutil
    import tempfile
    test_code = dedent("""
    from cython.parallel import parallel, prange, threadid
    cimport openmp
    def n_threads():
        with nogil, parallel():
            openmp.omp_get_num_threads()
    """)
    tmp_dir = tempfile.mkdtemp()
    fname = path.join(tmp_dir, 'check_omp.pyx')
    with open(fname, 'w') as fp:
        fp.write(test_code)
    extension = Extension(
        name='check_omp',
        sources=[fname],
        extra_compile_args=omp_compile_flags,
        extra_link_args=omp_link_flags,
    )
    has_omp = True
    try:
        mod = pyxbuild.pyx_to_dll(fname, extension, pyxbuild_dir=tmp_dir)
        print("-" * 70)
        print("Using OpenMP.")
        print("-" * 70)
    except CompileError:
        print("*" * 70)
        print("Unable to compile OpenMP code. Not using OpenMP.")
        print("*" * 70)
        has_omp = False
    except LinkError:
        print("*" * 70)
        print("Unable to link OpenMP code. Not using OpenMP.")
        print("*" * 70)
        has_omp = False
    finally:
        shutil.rmtree(tmp_dir)

    if has_omp:
        return omp_compile_flags, omp_link_flags, True
    else:
        return [], [], False
Ejemplo n.º 11
0
def get_omp_flags():
    """Returns openmp flags if OpenMP is available.

    Implementation based on https://bitbucket.org/pysph/pysph

    """
    omp_compile_flags, omp_link_flags = ['-fopenmp'], ['-fopenmp']

    env_var = os.environ.get('USE_OPENMP', '')
    if env_var.lower() in ['0', 'false', 'n']:
        print("-"*70)
        print("OpenMP disabled. Enable using 'USE_OPENMP'")
        print("-"*70)
        return [], [], False

    from textwrap import dedent
    try:
        from Cython.Distutils import Extension
        from pyximport import pyxbuild
    except ImportError:
        print("Unable to import Cython, disabling OpenMP for now.")
        return [], [], False
    from distutils.errors import CompileError, LinkError
    import shutil
    import tempfile
    test_code = dedent("""
    from cython.parallel import parallel, prange, threadid
    cimport openmp
    def n_threads():
        with nogil, parallel():
            openmp.omp_get_num_threads()
    """)
    tmp_dir = tempfile.mkdtemp()
    fname = path.join(tmp_dir, 'check_omp.pyx')
    with open(fname, 'w') as fp:
        fp.write(test_code)
    extension = Extension(
        name='check_omp', sources=[fname],
        extra_compile_args=omp_compile_flags,
        extra_link_args=omp_link_flags,
    )
    has_omp = True
    try:
        mod = pyxbuild.pyx_to_dll(fname, extension, pyxbuild_dir=tmp_dir)
        print("-"*70)
        print("Using OpenMP.")
        print("-"*70)
    except CompileError:
        print("*"*70)
        print("Unable to compile OpenMP code. Not using OpenMP.")
        print("*"*70)
        has_omp = False
    except LinkError:
        print("*"*70)
        print("Unable to link OpenMP code. Not using OpenMP.")
        print("*"*70)
        has_omp = False
    finally:
        shutil.rmtree(tmp_dir)

    if has_omp:
        return omp_compile_flags, omp_link_flags, True
    else:
        return [], [], False
Ejemplo n.º 12
0
REPEAT_COUNT = 5

import re
import sys
import timeit
from time import time

from itertools import combinations
from functools import partial

try:
    from pyximport.pyxbuild import pyx_to_dll
except ImportError:
    pass
else:
    so_path = pyx_to_dll('acora/_acora.pyx')
    import sys, os.path
    sys.path.insert(0, os.path.dirname(so_path))

from acora import AcoraBuilder, BytesAcora, UnicodeAcora, PyAcora


def prepare_benchmark_data():
    s = (
        'bdfdaskdjfhaslkdhfsadhfklashdflabcasdabcdJAKHDBVDFLNFCBLSADHFCALKSJ'
        'jklhcnajskbhfasjhancfksjdfhbvaliuradefhzcbdegnashdgfbcjaabesdhgkfcnash'
        'fdkhbdegxcbgjsvdhabcabcfcgbnxahsdbgfbcakjsdhgnfcxsababcmdabe')
    s = s.lower() + s + s.upper()
    search_string = s * 1000

    all_keywords = [
Ejemplo n.º 13
0
REPEAT_COUNT = 5

import re
import sys
import timeit
from time import time

from itertools import combinations
from functools import partial

try:
    from pyximport.pyxbuild import pyx_to_dll
except ImportError:
    pass
else:
    so_path = pyx_to_dll('acora/_acora.pyx')
    import sys, os.path
    sys.path.insert(0, os.path.dirname(so_path))

from acora import AcoraBuilder, BytesAcora, UnicodeAcora, PyAcora


def prepare_benchmark_data():
    s = ('bdfdaskdjfhaslkdhfsadhfklashdflabcasdabcdJAKHDBVDFLNFCBLSADHFCALKSJ'
        'jklhcnajskbhfasjhancfksjdfhbvaliuradefhzcbdegnashdgfbcjaabesdhgkfcnash'
        'fdkhbdegxcbgjsvdhabcabcfcgbnxahsdbgfbcakjsdhgnfcxsababcmdabe')
    s = s.lower() + s + s.upper()
    search_string = s * 1000

    all_keywords = [
        'ab', 'abc', 'abcd', 'abcabc', 'ababc', 'ABBBC', 'ABCABC',
Ejemplo n.º 14
0
def run(script, args,
        build_dir=None,
        include_dirs=None,
        verbose=False):
    """
    The main MrHooker entry function.

    @param script The .pyx script file used to build the LD_PRELOAD library.
    @param args The command to execute as a subprocess.
    @param build_dir The build directory, into which shared libraries are
                    stored. If this is None, then the library will be built in
                    a temporary directory and discarded at exit. Otherwise, the
                    directory will keep the build directories.
    @param include_dirs List of include directories to search for Pyrex header
                        files (.pxd) and C header files.
    """

    if verbose:
        print "Script path:    ", script
        print "Command:        ", args
        if build_dir is None:
            print "Build directory:", build_dir, "(temporary directory)"
        else:
            print "Build directory:", build_dir
        print "Include directories:", include_dirs

    # Locate libpython, in the likely event that the child process does not
    # load Python itself.
    import sysconfig
    libpython = "/".join(sysconfig.get_config_vars("LIBPL", "LDLIBRARY"))

    # Locate the "_init_preload" shared library.
    import imp
    (file_, _init_preload, desc) = imp.find_module("_init_preload", __path__)

    tempdir = None
    if not build_dir:
        # Create a temporary directory to store the shared library in. We'll
        # delete it when we're done.
        import tempfile
        tempdir = tempfile.mkdtemp()
        build_dir = tempdir

    try:
        from pyximport import pyxbuild
        from Cython.Distutils.extension import Extension

        # Get the module name from the script file.
        module_name = os.path.splitext(os.path.basename(script))[0]

        # Make sure the include directories are absolute, since the extension
        # may be built in another directory.
        if include_dirs:
            include_dirs = map(os.path.abspath, include_dirs)

        # Create and build an Extension.
        ext = Extension(module_name, [script], include_dirs=include_dirs)
        out_fname = pyxbuild.pyx_to_dll(
            script, ext, build_in_temp=True, pyxbuild_dir=build_dir)

        env = os.environ.copy()
        env["LD_PRELOAD"] = " ".join([libpython, out_fname, _init_preload])
        env["MRHOOKER_MODULE"] = module_name
        if verbose:
            print "Executing command with:\n" + \
                  "    LD_PRELOAD      =", env["LD_PRELOAD"] + "\n" + \
                  "    MRHOOKER_MODULE =", module_name + "\n"

        import subprocess
        rc = subprocess.call(args, env=env)
        if verbose:
            print "Command completed with return code '%d'" % rc
        return rc
    finally:
        if tempdir:
            import shutil
            shutil.rmtree(tempdir)
Ejemplo n.º 15
0
def pyx_to_dll_wrapper(filename, ext=None, force_rebuild=0, build_in_temp=False, pyxbuild_dir=None, setup_args={}, reload_support=False):
    """
    compila il file .pyx passato; ha la stessa signature della funzione
    pyximport.pyx_to_dll ma utilizza, se presente, il file build.cfg posto
    nella stessa directory del file .pyx.

    il file .cfg deve definire un dizionario "cython" con, opzionale, una
    chiave per ogni file .pyx, esempio:

    cython = {
        'foo': {
        }
    }

    Il dizionario associato a 'foo' verrà utilizzato per la compilazione del
    file foo.pyx; questo dizionario può contenere le seguenti chiave:

        * include_dirs
        * library_dirs
        * libraries

    che hanno lo stesso significato dei corrispettivi argomenti nella classe
    distutils.core.Extension

        http://docs.python.org/distutils/apiref.html#distutils.core.Extension

    unica differenza, include_dirs viene utilizzato anche da cython come elenco
    di path dove cercare eventuali .pxd

    la configurazione di un modulo può essere specificata per le varie
    piattaforme dove verrà compilato aggiungendo ad ogni attributo i suffissi:
    _linux, _nt, _darwin

    Per ogni attributo possono essere presenti sia la versione specifica per la
    piattaforma che quella generica, in questo caso i due valori vengono
    concatenati:

    cython = {
        'foo': {
            'include_dirs': ('/usr/include',),
            'include_dirs_linux': ('/usr/local/include',),
        }
    }

    Le directory presenti in `lib_dir`/pxd/ vengono aggiunte automaticamente al
    path di ricerca di cython *e* alle include_dirs passate al compilatore, in
    questo modo possiamo avere in un solo posto i file .pxd insieme ai file .h
    necessari a cython.
    """
    from distutils.extension import Extension
    try:
        from pyximport.pyxbuild import pyx_to_dll
    except TypeError: # unsupported operand type(s) for +: 'NoneType' and 'str'
        # Questo è un workaround per una incompatibilità con python -O
        # dell'estensione a distutils di cython che cerca fare
        # Extension.__doc__ + "altra documentazione", solo
        # che con l'ottimizzazione accesa Extension.__doc__ è None.
        assert Extension.__doc__ is None
        Extension.__doc__ = ""
        from pyximport.pyxbuild import pyx_to_dll

    if not pxd_include_dirs:
        pxd_include_dirs.extend(x for x in glob(os.path.join(common.opts.libs, 'pxd', '*')) if os.path.isdir(x))

    cfgpath = os.path.join(os.path.dirname(filename), 'build.cfg')
    modname = os.path.splitext(os.path.basename(filename))[0]
    cfg = common.parseConfig(cfgpath).get('cython', {})
    conf = Conf(cfg.get(modname, {}))

    if ext is None:
        # il nome dell'estensione deve essere un dotted name relativo alla
        # directory dei sorgenti
        name = os.path.normpath(filename)[len(common.opts.source):]
        if name[0] in '/\\':
            name = name[1:]
        name = os.path.splitext(name)[0].replace('/', '.').replace('\\', '.')
        ext = Extension(name=name, sources=[filename])

    include_dirs = pxd_include_dirs + ( conf['include_dirs'] or [] )
    library_dirs = conf['library_dirs'] or []
    libraries = conf['libraries'] or []

    def _s(name, value):
        if hasattr(ext, name):
            setattr(ext, name, getattr(ext, name) + value)
        else:
            setattr(ext, name, value)

    _s('pyrex_include_dirs', pxd_include_dirs)
    _s('include_dirs', include_dirs)
    _s('library_dirs', library_dirs)
    _s('libraries', libraries)

    return pyx_to_dll(
        filename,
        ext=ext,
        force_rebuild=True,
        build_in_temp=build_in_temp,
        pyxbuild_dir=pyxbuild_dir,
        setup_args=setup_args,
        reload_support=reload_support,
    )
Ejemplo n.º 16
0
Archivo: bench.py Proyecto: amumu/acora
REPEAT_COUNT = 5

import re
import sys
import timeit
from time import time

from itertools import combinations
from functools import partial

try:
    from pyximport.pyxbuild import pyx_to_dll
except ImportError:
    pass
else:
    so_path = pyx_to_dll("acora/_acora.pyx")
    import sys, os.path

    sys.path.insert(0, os.path.dirname(so_path))

from acora import AcoraBuilder, BytesAcora, UnicodeAcora, PyAcora


def prepare_benchmark_data():
    s = (
        "bdfdaskdjfhaslkdhfsadhfklashdflabcasdabcdJAKHDBVDFLNFCBLSADHFCALKSJ"
        "jklhcnajskbhfasjhancfksjdfhbvaliuradefhzcbdegnashdgfbcjaabesdhgkfcnash"
        "fdkhbdegxcbgjsvdhabcabcfcgbnxahsdbgfbcakjsdhgnfcxsababcmdabe"
    )
    s = s.lower() + s + s.upper()
    search_string = s * 1000
Ejemplo n.º 17
0
def get_openmp_flags():
    """Returns any OpenMP related flags if OpenMP is avaiable on the system.
    """
    if sys.platform == 'win32':
        omp_compile_flags, omp_link_flags = ['/openmp'], []
    else:
        omp_compile_flags, omp_link_flags = ['-fopenmp'], ['-fopenmp']

    env_var = os.environ.get('USE_OPENMP', '')
    if env_var.lower() in ("0", 'false', 'n'):
        print("-"*70)
        print("OpenMP disabled by environment variable (USE_OPENMP).")
        return [], [], False

    from textwrap import dedent
    try:
        from Cython.Distutils import Extension
        from pyximport import pyxbuild
    except ImportError:
        print("Unable to import Cython, disabling OpenMP for now.")
        return [], [], False

    from distutils.errors import CompileError, LinkError
    import shutil
    import tempfile
    test_code = dedent("""
    from cython.parallel import parallel, prange, threadid
    cimport openmp
    def n_threads():
        with nogil, parallel():
            openmp.omp_get_num_threads()
    """)
    tmp_dir = tempfile.mkdtemp()
    fname = path.join(tmp_dir, 'check_omp.pyx')
    with open(fname, 'w') as fp:
        fp.write(test_code)
    extension = Extension(
        name='check_omp', sources=[fname],
        extra_compile_args=omp_compile_flags,
        extra_link_args=omp_link_flags,
    )
    has_omp = True
    try:
        pyxbuild.pyx_to_dll(fname, extension, pyxbuild_dir=tmp_dir)
        print("-"*70)
        print("Using OpenMP.")
        print("-"*70)
    except CompileError:
        print("*"*70)
        print("Unable to compile OpenMP code. Not using OpenMP.")
        print("*"*70)
        has_omp = False
    except LinkError:
        print("*"*70)
        print("Unable to link OpenMP code. Not using OpenMP.")
        print("*"*70)
        has_omp = False
    finally:
        shutil.rmtree(tmp_dir)

    if has_omp:
        return omp_compile_flags, omp_link_flags, True
    else:
        return [], [], False
Ejemplo n.º 18
0
    from fwrap.main import main

    sys.stderr.write("Python %s\n" % sys.version)
    sys.stderr.write("\n")

    # insert cython.py/Cython source directory into sys.path
    cython_dir = os.path.abspath(os.path.join(os.path.pardir, os.path.pardir))
    sys.path.insert(0, cython_dir)

    if 0:
        if options.with_refnanny:
            from pyximport.pyxbuild import pyx_to_dll

            libpath = pyx_to_dll(
                os.path.join("Cython", "Runtime", "refnanny.pyx"),
                build_in_temp=True,
                pyxbuild_dir=os.path.join(WORKDIR, "support"),
            )
            sys.path.insert(0, os.path.split(libpath)[0])
            CFLAGS.append("-DCYTHON_REFNANNY")

    # if 0
    test_bugs = False
    if options.tickets:
        for ticket_number in options.tickets:
            test_bugs = True
            cmd_args.append(".*T%s$" % ticket_number)
    if not test_bugs:
        for selector in cmd_args:
            if selector.startswith("bugs"):
                test_bugs = True