Пример #1
0
def parse_mysql_config_info(options, stdout):
    log.debug("# stdout: {0}".format(stdout))
    info = {}
    for option, line in zip(options, stdout.split('\n')):
        log.debug("# option: {0}".format(option))
        log.debug("# line: {0}".format(line))
        info[option] = line.strip()

    ver = info['version']
    if '-' in ver:
        ver, _ = ver.split('-', 2)

    info['version'] = tuple([int(v) for v in ver.split('.')[0:3]])
    libs = shlex.split(info['libs'])
    info['lib_dir'] = libs[0].replace('-L', '')
    info['libs'] = [ lib.replace('-l', '') for lib in libs[1:] ]
    log.debug("# info['libs']: ")
    for lib in info['libs']:
        log.debug("#   {0}".format(lib))
    libs = shlex.split(info['libs_r'])
    info['lib_r_dir'] = libs[0].replace('-L', '')
    info['libs_r'] = [ lib.replace('-l', '') for lib in libs[1:] ]

    info['include'] = [x.strip() for x in info['include'].split('-I')[1:]]

    return info
Пример #2
0
 def get_externals(self):
     cmd = ['svn', 'propget', 'svn:externals', self.path, '-R', '--xml']
     code, lines = _run_command(cmd, encoding="utf-8")
     if code:
         log.debug("svn propget failed")
         return []
     return parse_externals_xml(lines, prefix=os.path.abspath(self.path))
Пример #3
0
 def get_externals(self):
     cmd = ["svn", "propget", "svn:externals", self.path, "-R", "--xml"]
     code, lines = _run_command(cmd, encoding="utf-8")
     if code:
         log.debug("svn propget failed")
         return []
     return parse_externals_xml(lines, prefix=os.path.abspath(self.path))
Пример #4
0
    def get_test_roots(self):
        """ Find test directories, skipping nested dirs and anything marked
            to skip under [pytest]:norecursedirs
        """
        res = []
        no_recurse = []
        cfg = parse.get_pkg_cfg_parser()
        if cfg.has_section('pytest') and \
           cfg.has_option('pytest', 'norecursedirs'):
            [no_recurse.extend(glob.glob(os.path.join(os.getcwd(), i)))
             for i in cfg.get('pytest', 'norecursedirs').split()]
            no_recurse = [os.path.abspath(i) for i in no_recurse]

        test_dirs = []
        for (dirpath, dirnames, _) in os.walk(os.getcwd()):
            if CONFIG.test_dirname in dirnames:
                test_dirs.append(os.path.join(dirpath, CONFIG.test_dirname))
        test_dirs.sort(key=len)
        for i in test_dirs:
            try:
                for j in res + no_recurse:
                    if i.startswith(j):
                        raise ValueError
            except ValueError:
                pass
            else:
                res.append(i)
        log.debug("Test roots: {0}".format(res))
        return res
Пример #5
0
    def make_file(self, infiles, outfile, func, args,
                  exec_msg=None, skip_msg=None, level=1):
        """Special case of 'execute()' for operations that process one or
        more input files and generate one output file.  Works just like
        'execute()', except the operation is skipped and a different
        message printed if 'outfile' already exists and is newer than all
        files listed in 'infiles'.  If the command defined 'self.force',
        and it is true, then the command is unconditionally run -- does no
        timestamp checks.
        """
        if exec_msg is None:
            exec_msg = "generating %s from %s" % (outfile, ', '.join(infiles))
        if skip_msg is None:
            skip_msg = "skipping %s (inputs unchanged)" % outfile


        # Allow 'infiles' to be a single string
        if isinstance(infiles, str):
            infiles = (infiles,)
        elif not isinstance(infiles, (list, tuple)):
            raise TypeError(
                  "'infiles' must be a string, or a list or tuple of strings")

        # If 'outfile' must be regenerated (either because it doesn't
        # exist, is out-of-date, or the 'force' flag is true) then
        # perform the action that presumably regenerates it
        if self.force or dep_util.newer_group (infiles, outfile):
            self.execute(func, args, exec_msg, level)
        # Otherwise, print the "skip" message
        else:
            log.debug(skip_msg)
Пример #6
0
def query_vcvarsall(version, arch="x86"):
    """Launch vcvarsall.bat and read the settings from its environment
    """
    vcvarsall = find_vcvarsall(version)
    interesting = set(("include", "lib", "libpath", "path"))
    result = {}

    if vcvarsall is None:
        raise DistutilsPlatformError("Unable to find vcvarsall.bat")
    log.debug("Calling 'vcvarsall.bat %s' (version=%s)", arch, version)
    popen = subprocess.Popen('"%s" %s & set' % (vcvarsall, arch),
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE)

    stdout, stderr = popen.communicate()
    if popen.wait() != 0:
        raise DistutilsPlatformError(stderr.decode("mbcs"))

    stdout = stdout.decode("mbcs")
    for line in stdout.split("\n"):
        line = Reg.convert_mbcs(line)
        if '=' not in line:
            continue
        line = line.strip()
        key, value = line.split('=', 1)
        key = key.lower()
        if key in interesting:
            if value.endswith(os.pathsep):
                value = value[:-1]
            result[key] = removeDuplicates(value)

    if len(result) != len(interesting):
        raise ValueError(str(list(result.keys())))

    return result
Пример #7
0
    def create_static_lib(self, objects, output_libname,
                          output_dir=None, debug=0, target_lang=None):
        objects, output_dir = self._fix_object_args(objects, output_dir)

        output_filename = \
            self.library_filename(output_libname, output_dir=output_dir)

        if self._need_link(objects, output_filename):
            self.mkpath(os.path.dirname(output_filename))
            self.spawn(self.archiver +
                       [output_filename] +
                       objects + self.objects)

            # Not many Unices required ranlib anymore -- SunOS 4.x is, I
            # think the only major Unix that does.  Maybe we need some
            # platform intelligence here to skip ranlib if it's not
            # needed -- or maybe Python's configure script took care of
            # it for us, hence the check for leading colon.
            if self.ranlib:
                try:
                    self.spawn(self.ranlib + [output_filename])
                except DistutilsExecError as msg:
                    raise LibError(msg)
        else:
            log.debug("skipping %s (up-to-date)", output_filename)
Пример #8
0
def run_cython(source, depends=(), includes=(),
               destdir_c=None, destdir_h=None,
               wdir=None, force=False, VERSION=None):
    from glob import glob
    from distutils import log
    from distutils import dep_util
    from distutils.errors import DistutilsError
    target = os.path.splitext(source)[0]+'.c'
    cwd = os.getcwd()
    try:
        if wdir: os.chdir(wdir)
        alldeps = [source]
        for dep in depends:
            alldeps += glob(dep)
        if not (force or dep_util.newer_group(alldeps, target)):
            log.debug("skipping '%s' -> '%s' (up-to-date)",
                      source, target)
            return
    finally:
        os.chdir(cwd)
    if not chk_cython(VERSION):
        raise DistutilsError("requires Cython>=%s" % VERSION)
    log.info("cythonizing '%s' -> '%s'", source, target)
    from cythonize import cythonize
    err = cythonize(source,
                    includes=includes,
                    destdir_c=destdir_c,
                    destdir_h=destdir_h,
                    wdir=wdir)
    if err:
        raise DistutilsError(
            "Cython failure: '%s' -> '%s'" % (source, target))
Пример #9
0
def find_vcvarsall(version):
    """Find the vcvarsall.bat file

    At first it tries to find the productdir of VS 2008 in the registry. If
    that fails it falls back to the VS90COMNTOOLS env var.
    """
    vsbase = VS_BASE % version
    try:
        productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
                                   "productdir")
    except KeyError:
        log.debug("Unable to find productdir in registry")
        productdir = None

    if not productdir or not os.path.isdir(productdir):
        toolskey = "VS%0.f0COMNTOOLS" % version
        toolsdir = os.environ.get(toolskey, None)

        if toolsdir and os.path.isdir(toolsdir):
            productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
            productdir = os.path.abspath(productdir)
            if not os.path.isdir(productdir):
                log.debug("%s is not a valid directory" % productdir)
                return None
        else:
            log.debug("Env var %s is not set or invalid" % toolskey)
    if not productdir:
        log.debug("No productdir found")
        return None
    vcvarsall = os.path.join(productdir, "vcvarsall.bat")
    if os.path.isfile(vcvarsall):
        return vcvarsall
    log.debug("Unable to find vcvarsall.bat")
    return None
Пример #10
0
def get_extensions():

    med_sources = [str(os.path.join(UTIL_DIR, "median_utils.pyx")),
                   str(os.path.join(UTIL_DIR, "quick_select.c"))]

    include_dirs = ['numpy', UTIL_DIR]

    libraries = []

    ext_med = Extension(name=str('banzai.utils.median_utils'),
                        sources=med_sources,
                        include_dirs=include_dirs,
                        libraries=libraries,
                        language="c",
                        extra_compile_args=['-g', '-O3', '-funroll-loops', '-ffast-math'])

    has_openmp, outputs = check_openmp()
    if has_openmp:
        if setup_helpers.get_compiler_option() == 'msvc':
            ext_med.extra_compile_args.append('-openmp')
        else:
            ext_med.extra_compile_args.append('-fopenmp')
            ext_med.extra_link_args = ['-g', '-fopenmp']
    else:
        log.warn('OpenMP was not found. '
                 'banzai will be compiled without OpenMP. '
                 '(Use the "-v" option of setup.py for more details.)')
        log.debug(('(Start of OpenMP info)\n'
                   'compiler stdout:\n{0}\n'
                   'compiler stderr:\n{1}\n'
                   '(End of OpenMP info)').format(*outputs))

    return [ext_med]
Пример #11
0
    def run(self):
        # remove the build/temp.<plat> directory (unless it's already
        # gone)
        if os.path.exists(self.build_temp):
            remove_tree(self.build_temp, dry_run=self.dry_run)
        else:
            log.debug("'%s' does not exist -- can't clean it",
                      self.build_temp)

        if self.all:
            # remove build directories
            for directory in (self.build_lib,
                              self.bdist_base,
                              self.build_scripts):
                if os.path.exists(directory):
                    remove_tree(directory, dry_run=self.dry_run)
                else:
                    log.warn("'%s' does not exist -- can't clean it",
                             directory)

        # just for the heck of it, try to remove the base build directory:
        # we might have emptied it right now, but if not we don't care
        if not self.dry_run:
            try:
                os.rmdir(self.build_base)
                log.info("removing '%s'", self.build_base)
            except OSError:
                pass
Пример #12
0
    def build_extension(self, ext):
        sources = ext.sources
        if sources is None or type(sources) not in (ListType, TupleType):
            raise DistutilsSetupError, ("in 'ext_modules' option (extension '%s'), " + "'sources' must be present and must be " + 'a list of source filenames') % ext.name
        sources = list(sources)
        ext_path = self.get_ext_fullpath(ext.name)
        depends = sources + ext.depends
        if not (self.force or newer_group(depends, ext_path, 'newer')):
            log.debug("skipping '%s' extension (up-to-date)", ext.name)
            return
        else:
            log.info("building '%s' extension", ext.name)
            sources = self.swig_sources(sources, ext)
            extra_args = ext.extra_compile_args or []
            macros = ext.define_macros[:]
            for undef in ext.undef_macros:
                macros.append((undef,))

            objects = self.compiler.compile(sources, output_dir=self.build_temp, macros=macros, include_dirs=ext.include_dirs, debug=self.debug, extra_postargs=extra_args, depends=ext.depends)
            self._built_objects = objects[:]
            if ext.extra_objects:
                objects.extend(ext.extra_objects)
            extra_args = ext.extra_link_args or []
            language = ext.language or self.compiler.detect_language(sources)
            self.compiler.link_shared_object(objects, ext_path, libraries=self.get_libraries(ext), library_dirs=ext.library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=language)
            return
def chmod(path, mode):
    from distutils import log
    log.debug("changing mode of %s to %o", path, mode)
    try:
        _chmod(path, mode)
    except os.error, e:
        log.debug("chmod failed: %s", e)
Пример #14
0
    def get_test_roots(self):
        """ Find test directories, skipping nested dirs and anything marked
            to skip under [pytest]:norecursedirs
        """
        from path import path
        res = []
        no_recurse = []
        cfg = config.get_pkg_cfg_parser()
        if cfg.has_section('pytest') and \
           cfg.has_option('pytest', 'norecursedirs'):
            [no_recurse.extend(path.getcwd().glob(i)) for i in
             cfg.get('pytest', 'norecursedirs').split()]
            no_recurse = [i.abspath() for i in no_recurse]

        test_dirs = [i for i in
                     path.getcwd().walkdirs(CONFIG.test_dirname)]
        test_dirs.sort(key=len)
        for i in test_dirs:
            try:
                for j in res + no_recurse:
                    if i.startswith(j):
                        raise ValueError
            except ValueError:
                pass
            else:
                res.append(i)
        log.debug("Test roots: {0}".format(res))
        return res
Пример #15
0
    def test_non_ascii(self):
        # Issues #8663, #34421: test that non-encodable text is escaped with
        # backslashreplace error handler and encodable non-ASCII text is
        # output as is.
        for errors in ('strict', 'backslashreplace', 'surrogateescape',
                       'replace', 'ignore'):
            with self.subTest(errors=errors):
                stdout = io.TextIOWrapper(io.BytesIO(),
                                          encoding='cp437', errors=errors)
                stderr = io.TextIOWrapper(io.BytesIO(),
                                          encoding='cp437', errors=errors)
                old_threshold = log.set_threshold(log.DEBUG)
                try:
                    with swap_attr(sys, 'stdout', stdout), \
                         swap_attr(sys, 'stderr', stderr):
                        log.debug('Dεbug\tMėssãge')
                        log.fatal('Fαtal\tÈrrōr')
                finally:
                    log.set_threshold(old_threshold)

                stdout.seek(0)
                self.assertEqual(stdout.read().rstrip(),
                        'Dεbug\tM?ss?ge' if errors == 'replace' else
                        'Dεbug\tMssge' if errors == 'ignore' else
                        'Dεbug\tM\\u0117ss\\xe3ge')
                stderr.seek(0)
                self.assertEqual(stderr.read().rstrip(),
                        'Fαtal\t?rr?r' if errors == 'replace' else
                        'Fαtal\trrr' if errors == 'ignore' else
                        'Fαtal\t\\xc8rr\\u014dr')
Пример #16
0
def _find_vc2015():
    try:
        key = winreg.OpenKeyEx(
            winreg.HKEY_LOCAL_MACHINE,
            r"Software\Microsoft\VisualStudio\SxS\VC7",
            access=winreg.KEY_READ | winreg.KEY_WOW64_32KEY
        )
    except OSError:
        log.debug("Visual C++ is not registered")
        return None, None

    best_version = 0
    best_dir = None
    with key:
        for i in count():
            try:
                v, vc_dir, vt = winreg.EnumValue(key, i)
            except OSError:
                break
            if v and vt == winreg.REG_SZ and os.path.isdir(vc_dir):
                try:
                    version = int(float(v))
                except (ValueError, TypeError):
                    continue
                if version >= 14 and version > best_version:
                    best_version, best_dir = version, vc_dir
    return best_version, best_dir
Пример #17
0
def _writeclientdoc(doc, thing, forceload=0):
    """Write HTML documentation to a file in the current directory.
    """
    docmodule = pydoc.HTMLDoc.docmodule
    def strongarm(self, object, name=None, mod=None, *ignored):
        result = docmodule(self, object, name, mod, *ignored)

        # Grab all the aliases to pyclasses and create links.
        nonmembers = []
        push = nonmembers.append
        for k,v in inspect.getmembers(object, inspect.isclass):
            if inspect.getmodule(v) is not object and getattr(v,'typecode',None) is not None:
                push('<a href="%s.html">%s</a>: pyclass alias<br/>' %(v.__name__,k))

        result += self.bigsection('Aliases', '#ffffff', '#eeaa77', ''.join(nonmembers))
        return result

    pydoc.HTMLDoc.docmodule = strongarm
    try:
        object, name = pydoc.resolve(thing, forceload)
        page = pydoc.html.page(pydoc.describe(object), pydoc.html.document(object, name))
        name = os.path.join(doc, name + '.html')
        file = open(name, 'w')
        file.write(page)
        file.close()
    except (ImportError, pydoc.ErrorDuringImport), value:
        log.debug(str(value))
Пример #18
0
 def run(self):
     self.run_command('build')
     build_lib = self.get_finalized_command('build').build_lib
     old_pythonpath = None
     try:
         if 'PYTHONPATH' in os.environ:
             old_pythonpath = os.environ['PYTHONPATH']
             os.environ['PYTHONPATH'] = build_lib + ':' + old_pythonpath
         else:
             os.environ['PYTHONPATH'] = build_lib
         python_exec = os.path.normpath(sys.executable)
         log.info("running unit tests...")
         try:
             cmd = command(python_exec, "-m", "netsa.dist.run_unit_tests",
                           "-v", *self.distribution.netsa_unit_test_modules)
             log.debug("PYTHONPATH=%s %s", os.environ['PYTHONPATH'], cmd)
             run_parallel([cmd], stdout=sys.stdout, stderr=sys.stderr)
         except:
             log.info("running unit tests... failed")
             sys.exit(-1)
         log.info("running unit tests... success")
     finally:
         if old_pythonpath:
             os.environ['PYTHONPATH'] = old_pythonpath
         else:
             del os.environ['PYTHONPATH']
Пример #19
0
 def run(self):
     self.run_command('build')
     build_lib = self.get_finalized_command('build').build_lib
     old_pythonpath = None
     try:
         if 'PYTHONPATH' in os.environ:
             old_pythonpath = os.environ['PYTHONPATH']
             os.environ['PYTHONPATH'] = build_lib + ':' + old_pythonpath
         else:
             os.environ['PYTHONPATH'] = build_lib
         python_exec = os.path.normpath(sys.executable)
         log.info("running other tests...")
         errors = False
         for test_mod in self.distribution.netsa_other_test_modules:
             try:
                 cmd = command(python_exec, "-m", test_mod,
                               self.distribution.source_dir)
                 log.debug("PYTHONPATH=%s %s",
                           os.environ['PYTHONPATH'], cmd)
                 run_parallel([cmd], stdout=sys.stdout, stderr=sys.stderr)
             except:
                 errors = True
         if errors:
             log.info("running other tests... failed")
             sys.exit(-1)
         log.info("running other tests... success")
     finally:
         if old_pythonpath:
             os.environ['PYTHONPATH'] = old_pythonpath
         else:
             del os.environ['PYTHONPATH']
Пример #20
0
 def run(self):
     if self.all:
         for (filename, template) in self.distribution.netsa_version_files:
             log.info("removing %r", filename)
             if not self.dry_run:
                 try:
                     os.unlink(filename)
                 except:
                     log.debug("%r doesn't exist -- can't clean it",
                               filename)
         html_dir = self.get_finalized_command('gen_doc_html').gen_doc_html
         if os.path.exists(html_dir):
             remove_tree(html_dir, dry_run=self.dry_run)
         pdf_file = self.get_finalized_command('gen_doc_pdf').gen_doc_pdf
         log.info("removing %r", pdf_file)
         if not self.dry_run:
             try:
                 os.unlink(pdf_file)
             except:
                 log.debug("%r doesn't exist -- can't clean it",
                           pdf_file)
     latex_dir = self.get_finalized_command('gen_doc_pdf').gen_doc_latex
     extra_dir = self.get_finalized_command('gen_doc_pdf').gen_doc_extra
     if os.path.exists(latex_dir):
         remove_tree(latex_dir, dry_run=self.dry_run)
     if os.path.exists(extra_dir):
         remove_tree(extra_dir, dry_run=self.dry_run)
     clean.run(self)
Пример #21
0
 def run(self):
     self.run_command('build')
     build_lib = self.get_finalized_command('build').build_lib
     if self.distribution.netsa_doc_dir:
         self.mkpath(self.gen_doc_extra)
         gen_doc_config(self.distribution, self.gen_doc_extra)
         old_pythonpath = None
         try:
             if 'PYTHONPATH' in os.environ:
                 old_pythonpath = os.environ['PYTHONPATH']
                 os.environ['PYTHONPATH'] = (
                     self.gen_doc_extra + ':' + build_lib +
                     ':' + old_pythonpath)
             else:
                 os.environ['PYTHONPATH'] = (
                     self.gen_doc_extra + ":" + build_lib)
             cmd = command("sphinx-build",
                           "-b", "html",
                           "-d", self.gen_doc_extra,
                           os.path.join(self.distribution.source_dir,
                                        self.distribution.netsa_doc_dir),
                           self.gen_doc_html)
             log.info("generating HTML documentation with Sphinx")
             log.debug("PYTHONPATH=%s %s", os.environ['PYTHONPATH'], cmd)
             run_parallel([cmd], stdout=sys.stdout, stderr=sys.stderr)
         finally:
             if old_pythonpath:
                 os.environ['PYTHONPATH'] = old_pythonpath
             else:
                 del os.environ['PYTHONPATH']
Пример #22
0
def make_archive(base_name, format, root_dir = None, base_dir = None, verbose = 0, dry_run = 0, owner = None, group = None):
    save_cwd = os.getcwd()
    if root_dir is not None:
        log.debug("changing into '%s'", root_dir)
        base_name = os.path.abspath(base_name)
        if not dry_run:
            os.chdir(root_dir)
    if base_dir is None:
        base_dir = os.curdir
    kwargs = {'dry_run': dry_run}
    try:
        format_info = ARCHIVE_FORMATS[format]
    except KeyError:
        raise ValueError, "unknown archive format '%s'" % format

    func = format_info[0]
    for arg, val in format_info[1]:
        kwargs[arg] = val

    if format != 'zip':
        kwargs['owner'] = owner
        kwargs['group'] = group
    try:
        filename = func(base_name, base_dir, **kwargs)
    finally:
        if root_dir is not None:
            log.debug("changing back to '%s'", save_cwd)
            os.chdir(save_cwd)

    return filename
Пример #23
0
 def _run_command(self, cmd, env=None):
     log.debug('running [%s]' % (' '.join(cmd), ))
     try:
         return check_output(cmd, cwd=self.work_path, env=env)
     except Exception:
         log.error('command failed [%s] via [%s]' % (' '.join(cmd), self.work_path, ))
         raise
Пример #24
0
 def run(self):
   if self.all:
     if os.path.exists(self.build_doc):
       remove_tree(self.build_doc, dry_run=self.dry_run)
     else:
       log.debug("%s doesn't exist -- can't clean it", self.build_doc)
   _clean.run(self)
Пример #25
0
 def __build_java_lib(self, jlib):
   log.info("Building java code for hadoop-%s" % jlib.hadoop_vinfo)
   compile_cmd = "javac"
   if jlib.classpath:
     compile_cmd += " -classpath %s" % jlib.classpath
   else:
     log.warn("WARNING: could not set classpath, java code may not compile")
   class_dir = os.path.join(self.build_temp, "pipes-%s" % jlib.hadoop_vinfo)
   package_path = os.path.join(self.build_lib, "pydoop", jlib.jar_name)
   if not os.path.exists(class_dir):
     os.mkdir(class_dir)
   compile_cmd += " -d '%s'" % class_dir
   log.info("Compiling Java classes")
   for f in jlib.java_files:
     compile_cmd += " %s" % f
   log.debug("Command: %s", compile_cmd)
   ret = os.system(compile_cmd)
   if ret:
     raise DistutilsSetupError(
       "Error compiling java component.  Command: %s" % compile_cmd
       )
   package_cmd = "jar -cf %(package_path)s -C %(class_dir)s ./it" % {
     'package_path': package_path, 'class_dir': class_dir
     }
   log.info("Packaging Java classes")
   log.debug("Command: %s", package_cmd)
   ret = os.system(package_cmd)
   if ret:
     raise DistutilsSetupError(
       "Error packaging java component.  Command: %s" % package_cmd
       )
Пример #26
0
    def zap_pyfiles(self):
        log.info("Removing .py files from temporary directory")
        for base, dirs, files in walk_egg(self.bdist_dir):
            for name in files:
                path = os.path.join(base, name)

                if name.endswith('.py'):
                    log.debug("Deleting %s", path)
                    os.unlink(path)

                if base.endswith('__pycache__'):
                    path_old = path

                    pattern = r'(?P<name>.+)\.(?P<magic>[^.]+)\.pyc'
                    m = re.match(pattern, name)
                    path_new = os.path.join(
                        base, os.pardir, m.group('name') + '.pyc')
                    log.info(
                        "Renaming file from [%s] to [%s]"
                        % (path_old, path_new))
                    try:
                        os.remove(path_new)
                    except OSError:
                        pass
                    os.rename(path_old, path_new)
Пример #27
0
    def run (self):

        # Obviously have to build before we can install
        if not self.skip_build:
            self.run_command('build')

        # Run all sub-commands (at least those that need to be run)
        for cmd_name in self.get_sub_commands():
            self.run_command(cmd_name)

        if self.path_file:
            self.create_path_file()

        # write list of installed files, if requested.
        if self.record:
            outputs = self.get_outputs()
            if self.root:               # strip any package prefix
                root_len = len(self.root)
                for counter in xrange(len(outputs)):
                    outputs[counter] = outputs[counter][root_len:]
            self.execute(write_file,
                         (self.record, outputs),
                         "writing list of installed files to '%s'" %
                         self.record)

        sys_path = map(os.path.normpath, sys.path)
        sys_path = map(os.path.normcase, sys_path)
        install_lib = os.path.normcase(os.path.normpath(self.install_lib))
        if (self.warn_dir and
            not (self.path_file and self.install_path_file) and
            install_lib not in sys_path):
            log.debug(("modules installed to '%s', which is not in "
                       "Python's module search path (sys.path) -- "
                       "you'll have to change the search path yourself"),
                       self.install_lib)
Пример #28
0
 def run(self):
     if not self.source_dir:
         log.error('no source directory specified')
         return
     for root, dirs, files in os.walk(self.source_dir):
         targ_dir = os.path.join(
             root.replace(self.source_dir, self.build_dir), 'LC_MESSAGES')
         for name in files:
             base, ext = os.path.splitext(name)
             if ext.lower() != '.po':
                 continue
             src = os.path.join(root, name)
             targ = os.path.join(targ_dir, base + '.mo')
             if os.path.exists(targ):
                 targ_date = os.stat(targ)[8]
             else:
                 targ_date = 0
             src_date = os.stat(src)[8]
             if targ_date > src_date:
                 log.debug('skipping %s', targ)
                 continue
             self.mkpath(targ_dir)
             log.info('compiling %s to %s', src, targ)
             if not self.dry_run:
                 subprocess.check_call(
                     ['msgfmt', '--use-fuzzy',
                      '--output-file=%s' % targ, src])
Пример #29
0
 def find_sources(self):
     """Finds the sources as default and then drop the cruft."""
     vsc_egg_info.find_sources(self)
     for fn in remove_extra_bdist_rpm_files():
         log.debug("removing %s from source list" % (fn))
         if fn in self.filelist.files:
             self.filelist.files.remove(fn)
Пример #30
0
    def _get_dlls(self):
        # return a list of of (FQ-in-name, relative-out-name) tuples.
        ret = []
        # the apr binaries.
        apr_bins = [libname + ".dll" for libname in extra_libs
                    if libname.startswith("libapr")]
        if get_svn_version() >= (1,5,0):
            # Since 1.5.0 these libraries became shared
            apr_bins += """libsvn_client-1.dll libsvn_delta-1.dll libsvn_diff-1.dll
                           libsvn_fs-1.dll libsvn_ra-1.dll libsvn_repos-1.dll
                           libsvn_subr-1.dll libsvn_wc-1.dll libsasl.dll""".split()
        if get_svn_version() >= (1,7,0):
            apr_bins += ["libdb48.dll"]
        else:
            apr_bins += ["libdb44.dll"]
        apr_bins += """intl3_svn.dll libeay32.dll ssleay32.dll""".split()
        look_dirs = os.environ.get("PATH","").split(os.pathsep)
        look_dirs.insert(0, os.path.join(os.environ["SVN_DEV"], "bin"))

        target = os.path.abspath(os.path.join(self.build_lib, 'subvertpy'))
        for bin in apr_bins:
            for look in look_dirs:
                f = os.path.join(look, bin)
                if os.path.isfile(f):
                    ret.append((f, target))
                    break
            else:
                log.warn("Could not find required DLL %r to include", bin)
                log.debug("(looked in %s)", look_dirs)
        return ret
Пример #31
0
    def copy_scripts(self):
        """Copy each script listed in 'self.scripts'; if it's marked as a
        Python script in the Unix way (first line matches 'first_line_re',
        ie. starts with "\#!" and contains "python"), then adjust the first
        line to refer to the current Python interpreter as we copy.
        """
        self.mkpath(self.build_dir)
        outfiles = []
        for script in self.scripts:
            adjust = 0
            script = convert_path(script)
            outfile = os.path.join(self.build_dir, os.path.basename(script))
            outfiles.append(outfile)

            if not self.force and not newer(script, outfile):
                log.debug("not copying %s (up-to-date)", script)
                continue

            # Always open the file, but ignore failures in dry-run mode --
            # that way, we'll get accurate feedback if we can read the
            # script.
            try:
                f = open(script, "r")
            except IOError:
                if not self.dry_run:
                    raise
                f = None
            else:
                first_line = f.readline()
                if not first_line:
                    self.warn("%s is an empty file (skipping)" % script)
                    continue

                match = first_line_re.match(first_line)
                if match:
                    adjust = 1
                    post_interp = match.group(1) or ''

            if adjust:
                log.info("copying and adjusting %s -> %s", script,
                         self.build_dir)
                if not self.dry_run:
                    outf = open(outfile, "w")
                    if not sysconfig.python_build:
                        outf.write("#!%s%s\n" % (self.executable, post_interp))
                    else:
                        outf.write("#!%s%s\n" % (os.path.join(
                            sysconfig.get_config_var("BINDIR"),
                            "python" + sysconfig.get_config_var("VERSION") +
                            sysconfig.get_config_var("EXE")), post_interp))
                    outf.writelines(f.readlines())
                    outf.close()
                if f:
                    f.close()
            else:
                if f:
                    f.close()
                self.copy_file(script, outfile)

        if os.name == 'posix':
            for file in outfiles:
                if self.dry_run:
                    log.info("changing mode of %s", file)
                else:
                    oldmode = os.stat(file)[ST_MODE] & 07777
                    newmode = (oldmode | 0555) & 07777
                    if newmode != oldmode:
                        log.info("changing mode of %s from %o to %o", file,
                                 oldmode, newmode)
                        os.chmod(file, newmode)
    def link(self,
             target_desc,
             objects,
             output_filename,
             output_dir=None,
             libraries=None,
             library_dirs=None,
             runtime_library_dirs=None,
             export_symbols=None,
             debug=0,
             extra_preargs=None,
             extra_postargs=None,
             build_temp=None,
             target_lang=None):

        # XXX this ignores 'build_temp'!  should follow the lead of
        # msvccompiler.py

        (objects, output_dir) = self._fix_object_args(objects, output_dir)
        (libraries, library_dirs, runtime_library_dirs) = \
            self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)

        if runtime_library_dirs:
            log.warn("I don't know what to do with 'runtime_library_dirs': %s",
                     str(runtime_library_dirs))

        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)

        if self._need_link(objects, output_filename):

            # Figure out linker args based on type of target.
            if target_desc == CCompiler.EXECUTABLE:
                startup_obj = 'c0w32'
                if debug:
                    ld_args = self.ldflags_exe_debug[:]
                else:
                    ld_args = self.ldflags_exe[:]
            else:
                startup_obj = 'c0d32'
                if debug:
                    ld_args = self.ldflags_shared_debug[:]
                else:
                    ld_args = self.ldflags_shared[:]

            # Create a temporary exports file for use by the linker
            if export_symbols is None:
                def_file = ''
            else:
                head, tail = os.path.split(output_filename)
                modname, ext = os.path.splitext(tail)
                temp_dir = os.path.dirname(
                    objects[0])  # preserve tree structure
                def_file = os.path.join(temp_dir, '%s.def' % modname)
                contents = ['EXPORTS']
                for sym in (export_symbols or []):
                    contents.append('  %s=_%s' % (sym, sym))
                self.execute(write_file, (def_file, contents),
                             "writing %s" % def_file)

            # Borland C++ has problems with '/' in paths
            objects2 = map(os.path.normpath, objects)
            # split objects in .obj and .res files
            # Borland C++ needs them at different positions in the command line
            objects = [startup_obj]
            resources = []
            for file in objects2:
                (base, ext) = os.path.splitext(os.path.normcase(file))
                if ext == '.res':
                    resources.append(file)
                else:
                    objects.append(file)

            for l in library_dirs:
                ld_args.append("/L%s" % os.path.normpath(l))
            ld_args.append("/L.")  # we sometimes use relative paths

            # list of object files
            ld_args.extend(objects)

            # XXX the command-line syntax for Borland C++ is a bit wonky;
            # certain filenames are jammed together in one big string, but
            # comma-delimited.  This doesn't mesh too well with the
            # Unix-centric attitude (with a DOS/Windows quoting hack) of
            # 'spawn()', so constructing the argument list is a bit
            # awkward.  Note that doing the obvious thing and jamming all
            # the filenames and commas into one argument would be wrong,
            # because 'spawn()' would quote any filenames with spaces in
            # them.  Arghghh!.  Apparently it works fine as coded...

            # name of dll/exe file
            ld_args.extend([',', output_filename])
            # no map file and start libraries
            ld_args.append(',,')

            for lib in libraries:
                # see if we find it and if there is a bcpp specific lib
                # (xxx_bcpp.lib)
                libfile = self.find_library_file(library_dirs, lib, debug)
                if libfile is None:
                    ld_args.append(lib)
                    # probably a BCPP internal library -- don't warn
                else:
                    # full name which prefers bcpp_xxx.lib over xxx.lib
                    ld_args.append(libfile)

            # some default libraries
            ld_args.append('import32')
            ld_args.append('cw32mt')

            # def file for export symbols
            ld_args.extend([',', def_file])
            # add resource files
            ld_args.append(',')
            ld_args.extend(resources)

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)

            self.mkpath(os.path.dirname(output_filename))
            try:
                self.spawn([self.linker] + ld_args)
            except DistutilsExecError as msg:
                raise LinkError(msg)

        else:
            log.debug("skipping %s (up-to-date)", output_filename)
Пример #33
0
    def build_extension(self, ext):
        sources = ext.sources
        if sources is None or type(sources) not in (ListType, TupleType):
            raise DistutilsSetupError, \
                  ("in 'ext_modules' option (extension '%s'), " +
                   "'sources' must be present and must be " +
                   "a list of source filenames") % ext.name
        sources = list(sources)

        fullname = self.get_ext_fullname(ext.name)
        if self.inplace:
            # ignore build-lib -- put the compiled extension into
            # the source tree along with pure Python modules

            modpath = string.split(fullname, '.')
            package = string.join(modpath[0:-1], '.')
            base = modpath[-1]

            build_py = self.get_finalized_command('build_py')
            package_dir = build_py.get_package_dir(package)
            ext_filename = os.path.join(package_dir,
                                        self.get_ext_filename(base))
        else:
            ext_filename = os.path.join(self.build_lib,
                                        self.get_ext_filename(fullname))
        depends = sources + ext.depends
        if not (self.force or newer_group(depends, ext_filename, 'newer')):
            log.debug("skipping '%s' extension (up-to-date)", ext.name)
            return
        else:
            log.info("building '%s' extension", ext.name)

        # First, scan the sources for SWIG definition files (.i), run
        # SWIG on 'em to create .c files, and modify the sources list
        # accordingly.
        sources = self.swig_sources(sources, ext)

        # Next, compile the source code to object files.

        # XXX not honouring 'define_macros' or 'undef_macros' -- the
        # CCompiler API needs to change to accommodate this, and I
        # want to do one thing at a time!

        # Two possible sources for extra compiler arguments:
        #   - 'extra_compile_args' in Extension object
        #   - CFLAGS environment variable (not particularly
        #     elegant, but people seem to expect it and I
        #     guess it's useful)
        # The environment variable should take precedence, and
        # any sensible compiler will give precedence to later
        # command line args.  Hence we combine them in order:
        extra_args = ext.extra_compile_args or []

        macros = ext.define_macros[:]
        for undef in ext.undef_macros:
            macros.append((undef, ))

        objects = self.compiler.compile(sources,
                                        output_dir=self.build_temp,
                                        macros=macros,
                                        include_dirs=ext.include_dirs,
                                        debug=self.debug,
                                        extra_postargs=extra_args,
                                        depends=ext.depends)

        # XXX -- this is a Vile HACK!
        #
        # The setup.py script for Python on Unix needs to be able to
        # get this list so it can perform all the clean up needed to
        # avoid keeping object files around when cleaning out a failed
        # build of an extension module.  Since Distutils does not
        # track dependencies, we have to get rid of intermediates to
        # ensure all the intermediates will be properly re-built.
        #
        self._built_objects = objects[:]

        # Now link the object files together into a "shared object" --
        # of course, first we have to figure out all the other things
        # that go into the mix.
        if ext.extra_objects:
            objects.extend(ext.extra_objects)
        extra_args = ext.extra_link_args or []

        # Detect target language, if not provided
        language = ext.language or self.compiler.detect_language(sources)

        self.compiler.link_shared_object(
            objects,
            ext_filename,
            libraries=self.get_libraries(ext),
            library_dirs=ext.library_dirs,
            runtime_library_dirs=ext.runtime_library_dirs,
            extra_postargs=extra_args,
            export_symbols=self.get_export_symbols(ext),
            debug=self.debug,
            build_temp=self.build_temp,
            target_lang=language)
Пример #34
0
    def link_osx(self,
                 target_desc,
                 objects,
                 output_filename,
                 output_dir=None,
                 libraries=None,
                 library_dirs=None,
                 runtime_library_dirs=None,
                 export_symbols=None,
                 debug=0,
                 extra_preargs=None,
                 extra_postargs=None,
                 build_temp=None,
                 target_lang=None):
        """Link the objects."""
        # use separate copies, so we can modify the lists
        libraries = copy.copy(libraries or [])
        libraries.extend(self.dll_libraries)

        # Copy of unix link so we can remove the darwin test
        #UnixCCompiler.link(self, target_desc, objects, output_filename,
        #    output_dir, libraries, library_dirs, runtime_library_dirs,
        #    None, # export_symbols, we do this in our def-file
        #    debug, extra_preargs, extra_postargs, build_temp, target_lang)

        objects, output_dir = self._fix_object_args(objects, output_dir)
        libraries, library_dirs, runtime_library_dirs = \
            self._fix_lib_args(libraries, library_dirs, runtime_library_dirs)

        lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
                                   libraries)
        #if type(output_dir) not in (StringType, NoneType):
        #    raise TypeError("'output_dir' must be a string or None")
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)

        if self._need_link(objects, output_filename):
            ld_args = (objects + self.objects + lib_opts +
                       ['-o', output_filename])
            if debug:
                ld_args[:0] = ['-g']
            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)
            self.mkpath(os.path.dirname(output_filename))
            try:
                if target_desc == CCompiler.EXECUTABLE:
                    linker = self.linker_exe[:]
                else:
                    linker = self.linker_so[:]
                if target_lang == "c++" and self.compiler_cxx:
                    # skip over environment variable settings if /usr/bin/env
                    # is used to set up the linker's environment.
                    # This is needed on OSX. Note: this assumes that the
                    # normal and C++ compiler have the same environment
                    # settings.
                    i = 0
                    if os.path.basename(linker[0]) == "env":
                        i = 1
                        while '=' in linker[i]:
                            i = i + 1

                    linker[i] = self.compiler_cxx[i]

                #if sys.platform == 'darwin':
                #    linker = _osx_support.compiler_fixup(linker, ld_args)
                #    ld_args = ['-arch', 'x86_64'] + ld_args

                self.spawn(linker + ld_args)
            except DistutilsExecError as msg:
                raise LinkError(msg)
        else:
            log.debug("skipping %s (up-to-date)", output_filename)
Пример #35
0
    def build_extension(self, ext):
        sources = ext.sources
        if sources is None or not isinstance(sources, (list, tuple)):
            raise DistutilsSetupError(
                "in 'ext_modules' option (extension '%s'), "
                "'sources' must be present and must be "
                "a list of source filenames" % ext.name)
        sources = list(sources)

        ext_path = self.get_ext_fullpath(ext.name)
        depends = sources + ext.depends
        if not (self.force or newer_group(depends, ext_path, 'newer')):
            log.debug("skipping '%s' extension (up-to-date)", ext.name)
            return
        else:
            log.info("building '%s' extension", ext.name)

        # First, scan the sources for SWIG definition files (.i), run
        # SWIG on 'em to create .c files, and modify the sources list
        # accordingly.
        sources = self.swig_sources(sources, ext)

        # Next, compile the source code to object files.

        # XXX not honouring 'define_macros' or 'undef_macros' -- the
        # CCompiler API needs to change to accommodate this, and I
        # want to do one thing at a time!

        # Two possible sources for extra compiler arguments:
        #   - 'extra_compile_args' in Extension object
        #   - CFLAGS environment variable (not particularly
        #     elegant, but people seem to expect it and I
        #     guess it's useful)
        # The environment variable should take precedence, and
        # any sensible compiler will give precedence to later
        # command line args.  Hence we combine them in order:
        extra_args = ext.extra_compile_args or []

        macros = ext.define_macros[:]
        for undef in ext.undef_macros:
            macros.append((undef, ))

        objects = self.compiler.compile(sources,
                                        output_dir=self.build_temp,
                                        macros=macros,
                                        include_dirs=ext.include_dirs,
                                        debug=self.debug,
                                        extra_postargs=extra_args,
                                        depends=ext.depends)

        # XXX outdated variable, kept here in case third-part code
        # needs it.
        self._built_objects = objects[:]

        # Now link the object files together into a "shared object" --
        # of course, first we have to figure out all the other things
        # that go into the mix.
        if ext.extra_objects:
            objects.extend(ext.extra_objects)
        extra_args = ext.extra_link_args or []

        # Detect target language, if not provided
        language = ext.language or self.compiler.detect_language(sources)

        self.compiler.link_shared_object(
            objects,
            ext_path,
            libraries=self.get_libraries(ext),
            library_dirs=ext.library_dirs,
            runtime_library_dirs=ext.runtime_library_dirs,
            extra_postargs=extra_args,
            export_symbols=self.get_export_symbols(ext),
            debug=self.debug,
            build_temp=self.build_temp,
            target_lang=language)
Пример #36
0
    def link(self, target_desc, objects,
             output_filename, output_dir=None, libraries=None,
             library_dirs=None, runtime_library_dirs=None,
             export_symbols=None, debug=0, extra_preargs=None,
             extra_postargs=None, build_temp=None, target_lang=None):
        objects, output_dir = self._fix_object_args(objects, output_dir)
        fixed_args = self._fix_lib_args(libraries, library_dirs,
                                        runtime_library_dirs)
        libraries, library_dirs, runtime_library_dirs = fixed_args

        # filter out standard library paths, which are not explicitely needed
        # for linking
        system_libdirs = ['/lib', '/lib64', '/usr/lib', '/usr/lib64']
        multiarch = sysconfig.get_config_var("MULTIARCH")
        if multiarch:
            system_libdirs.extend(['/lib/%s' % multiarch, '/usr/lib/%s' % multiarch])
        library_dirs = [dir for dir in library_dirs
                        if not dir in system_libdirs]
        runtime_library_dirs = [dir for dir in runtime_library_dirs
                                if not dir in system_libdirs]

        lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
                                   libraries)
        if not isinstance(output_dir, (str, type(None))):
            raise TypeError("'output_dir' must be a string or None")
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)

        if self._need_link(objects, output_filename):
            ld_args = (objects + self.objects +
                       lib_opts + ['-o', output_filename])
            if debug:
                ld_args[:0] = ['-g']
            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)
            self.mkpath(os.path.dirname(output_filename))
            try:
                if target_desc == CCompiler.EXECUTABLE:
                    linker = self.linker_exe[:]
                else:
                    linker = self.linker_so[:]
                if target_lang == "c++" and self.compiler_cxx:
                    # skip over environment variable settings if /usr/bin/env
                    # is used to set up the linker's environment.
                    # This is needed on OSX. Note: this assumes that the
                    # normal and C++ compiler have the same environment
                    # settings.
                    i = 0
                    if os.path.basename(linker[0]) == "env":
                        i = 1
                        while '=' in linker[i]:
                            i += 1
                    linker[i] = self.compiler_cxx[i]

                if sys.platform == 'darwin':
                    linker = _osx_support.compiler_fixup(linker, ld_args)

                self.spawn(linker + ld_args)
            except DistutilsExecError as msg:
                raise LinkError(msg)
        else:
            log.debug("skipping %s (up-to-date)", output_filename)
Пример #37
0
 def announce(self, msg, level=1):
     log.debug(msg)
Пример #38
0
    def link(self,
             target_desc,
             objects,
             output_filename,
             output_dir=None,
             libraries=None,
             library_dirs=None,
             runtime_library_dirs=None,
             export_symbols=None,
             debug=0,
             extra_preargs=None,
             extra_postargs=None,
             build_temp=None,
             target_lang=None):

        if not self.initialized:
            self.initialize()
        (objects, output_dir) = self._fix_object_args(objects, output_dir)
        fixed_args = self._fix_lib_args(libraries, library_dirs,
                                        runtime_library_dirs)
        (libraries, library_dirs, runtime_library_dirs) = fixed_args

        if runtime_library_dirs:
            self.warn ("I don't know what to do with 'runtime_library_dirs': "
                       + str (runtime_library_dirs))

        lib_opts = gen_lib_options(self,
                                   library_dirs, runtime_library_dirs,
                                   libraries)
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)

        if self._need_link(objects, output_filename):
            if target_desc == CCompiler.EXECUTABLE:
                if debug:
                    ldflags = self.ldflags_shared_debug[1:]
                else:
                    ldflags = self.ldflags_shared[1:]
            else:
                if debug:
                    ldflags = self.ldflags_shared_debug
                else:
                    ldflags = self.ldflags_shared

            export_opts = []
            for sym in (export_symbols or []):
                export_opts.append("/EXPORT:" + sym)

            ld_args = (ldflags + lib_opts + export_opts +
                       objects + ['/OUT:' + output_filename])

            # The MSVC linker generates .lib and .exp files, which cannot be
            # suppressed by any linker switches. The .lib files may even be
            # needed! Make sure they are generated in the temporary build
            # directory. Since they have different names for debug and release
            # builds, they can go into the same directory.
            if export_symbols is not None:
                (dll_name, dll_ext) = os.path.splitext(
                    os.path.basename(output_filename))
                implib_file = os.path.join(
                    os.path.dirname(objects[0]),
                    self.library_filename(dll_name))
                ld_args.append ('/IMPLIB:' + implib_file)

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)

            self.mkpath(os.path.dirname(output_filename))
            try:
                self.spawn([self.linker] + ld_args)
            except DistutilsExecError as msg:
                raise LinkError(msg)

        else:
            log.debug("skipping %s (up-to-date)", output_filename)
Пример #39
0
def byte_compile(
    py_files,
    optimize=0,
    force=0,
    prefix=None,
    base_dir=None,
    verbose=1,
    dry_run=0,
    direct=None,
):
    """Byte-compile a collection of Python source files to .pyc
    files in a __pycache__ subdirectory.  'py_files' is a list
    of files to compile; any files that don't end in ".py" are silently
    skipped.  'optimize' must be one of the following:
      0 - don't optimize
      1 - normal optimization (like "python -O")
      2 - extra optimization (like "python -OO")
    If 'force' is true, all files are recompiled regardless of
    timestamps.

    The source filename encoded in each bytecode file defaults to the
    filenames listed in 'py_files'; you can modify these with 'prefix' and
    'basedir'.  'prefix' is a string that will be stripped off of each
    source filename, and 'base_dir' is a directory name that will be
    prepended (after 'prefix' is stripped).  You can supply either or both
    (or neither) of 'prefix' and 'base_dir', as you wish.

    If 'dry_run' is true, doesn't actually do anything that would
    affect the filesystem.

    Byte-compilation is either done directly in this interpreter process
    with the standard py_compile module, or indirectly by writing a
    temporary script and executing it.  Normally, you should let
    'byte_compile()' figure out to use direct compilation or not (see
    the source for details).  The 'direct' flag is used by the script
    generated in indirect mode; unless you know what you're doing, leave
    it set to None.
    """

    # nothing is done if sys.dont_write_bytecode is True
    if sys.dont_write_bytecode:
        raise DistutilsByteCompileError('byte-compiling is disabled.')

    # First, if the caller didn't force us into direct or indirect mode,
    # figure out which mode we should be in.  We take a conservative
    # approach: choose direct mode *only* if the current interpreter is
    # in debug mode and optimize is 0.  If we're not in debug mode (-O
    # or -OO), we don't know which level of optimization this
    # interpreter is running with, so we can't do direct
    # byte-compilation and be certain that it's the right thing.  Thus,
    # always compile indirectly if the current interpreter is in either
    # optimize mode, or if either optimization level was requested by
    # the caller.
    if direct is None:
        direct = __debug__ and optimize == 0

    # "Indirect" byte-compilation: write a temporary script and then
    # run it with the appropriate flags.
    if not direct:
        try:
            from tempfile import mkstemp

            (script_fd, script_name) = mkstemp(".py")
        except ImportError:
            from tempfile import mktemp

            (script_fd, script_name) = None, mktemp(".py")
        log.info("writing byte-compilation script '%s'", script_name)
        if not dry_run:
            if script_fd is not None:
                script = os.fdopen(script_fd, "w")
            else:
                script = open(script_name, "w")

            with script:
                script.write("""\
from distutils.util import byte_compile
files = [
""")

                # XXX would be nice to write absolute filenames, just for
                # safety's sake (script should be more robust in the face of
                # chdir'ing before running it).  But this requires abspath'ing
                # 'prefix' as well, and that breaks the hack in build_lib's
                # 'byte_compile()' method that carefully tacks on a trailing
                # slash (os.sep really) to make sure the prefix here is "just
                # right".  This whole prefix business is rather delicate -- the
                # problem is that it's really a directory, but I'm treating it
                # as a dumb string, so trailing slashes and so forth matter.

                script.write(",\n".join(map(repr, py_files)) + "]\n")
                script.write("""
byte_compile(files, optimize=%r, force=%r,
             prefix=%r, base_dir=%r,
             verbose=%r, dry_run=0,
             direct=1)
""" % (optimize, force, prefix, base_dir, verbose))

        cmd = [sys.executable]
        cmd.extend(subprocess._optim_args_from_interpreter_flags())
        cmd.append(script_name)
        spawn(cmd, dry_run=dry_run)
        execute(os.remove, (script_name, ),
                "removing %s" % script_name,
                dry_run=dry_run)

    # "Direct" byte-compilation: use the py_compile module to compile
    # right here, right now.  Note that the script generated in indirect
    # mode simply calls 'byte_compile()' in direct mode, a weird sort of
    # cross-process recursion.  Hey, it works!
    else:
        from py_compile import compile

        for file in py_files:
            if file[-3:] != ".py":
                # This lets us be lazy and not filter filenames in
                # the "install_lib" command.
                continue

            # Terminology from the py_compile module:
            #   cfile - byte-compiled file
            #   dfile - purported source filename (same as 'file' by default)
            if optimize >= 0:
                opt = '' if optimize == 0 else optimize
                cfile = importlib.util.cache_from_source(file,
                                                         optimization=opt)
            else:
                cfile = importlib.util.cache_from_source(file)
            dfile = file
            if prefix:
                if file[:len(prefix)] != prefix:
                    raise ValueError(
                        "invalid prefix: filename %r doesn't start with %r" %
                        (file, prefix))
                dfile = dfile[len(prefix):]
            if base_dir:
                dfile = os.path.join(base_dir, dfile)

            cfile_base = os.path.basename(cfile)
            if direct:
                if force or newer(file, cfile):
                    log.info("byte-compiling %s to %s", file, cfile_base)
                    if not dry_run:
                        compile(file, cfile, dfile)
                else:
                    log.debug("skipping byte-compilation of %s to %s", file,
                              cfile_base)
Пример #40
0
    def link(self,
             target_desc,
             objects,
             output_filename,
             output_dir=None,
             libraries=None,
             library_dirs=None,
             runtime_library_dirs=None,
             export_symbols=None,
             debug=0,
             extra_preargs=None,
             extra_postargs=None,
             build_temp=None,
             target_lang=None):

        if not self.initialized:
            self.initialize()
        (objects, output_dir) = self._fix_object_args(objects, output_dir)
        fixed_args = self._fix_lib_args(libraries, library_dirs,
                                        runtime_library_dirs)
        (libraries, library_dirs, runtime_library_dirs) = fixed_args

        if runtime_library_dirs:
            self.warn ("I don't know what to do with 'runtime_library_dirs': "
                       + str (runtime_library_dirs))

        lib_opts = gen_lib_options(self,
                                   library_dirs, runtime_library_dirs,
                                   libraries)
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)

        if self._need_link(objects, output_filename):
            if target_desc == CCompiler.EXECUTABLE:
                if debug:
                    ldflags = self.ldflags_shared_debug[1:]
                else:
                    ldflags = self.ldflags_shared[1:]
            else:
                if debug:
                    ldflags = self.ldflags_shared_debug
                else:
                    ldflags = self.ldflags_shared

            export_opts = []
            for sym in (export_symbols or []):
                export_opts.append("/EXPORT:" + sym)

            ld_args = (ldflags + lib_opts + export_opts +
                       objects + ['/OUT:' + output_filename])

            # The MSVC linker generates .lib and .exp files, which cannot be
            # suppressed by any linker switches. The .lib files may even be
            # needed! Make sure they are generated in the temporary build
            # directory. Since they have different names for debug and release
            # builds, they can go into the same directory.
            build_temp = os.path.dirname(objects[0])
            if export_symbols is not None:
                (dll_name, dll_ext) = os.path.splitext(
                    os.path.basename(output_filename))
                implib_file = os.path.join(
                    build_temp,
                    self.library_filename(dll_name))
                ld_args.append ('/IMPLIB:' + implib_file)

            self.manifest_setup_ldargs(output_filename, build_temp, ld_args)

            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)

            self.mkpath(os.path.dirname(output_filename))
            try:
                self.spawn([self.linker] + ld_args)
            except DistutilsExecError as msg:
                raise LinkError(msg)

            # embed the manifest
            # XXX - this is somewhat fragile - if mt.exe fails, distutils
            # will still consider the DLL up-to-date, but it will not have a
            # manifest.  Maybe we should link to a temp file?  OTOH, that
            # implies a build environment error that shouldn't go undetected.
            mfinfo = self.manifest_get_embed_info(target_desc, ld_args)
            if mfinfo is not None:
                mffilename, mfid = mfinfo
                out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
                try:
                    self.spawn(['mt.exe', '-nologo', '-manifest',
                                mffilename, out_arg])
                except DistutilsExecError as msg:
                    raise LinkError(msg)
        else:
            log.debug("skipping %s (up-to-date)", output_filename)
Пример #41
0
    def build_dlib():
        """use cmake to build and install the extension
        """
        if cmake_path is None:
            cmake_install_url = "https://cmake.org/install/"
            message = ("You can install cmake using the instructions at " +
                       cmake_install_url)
            msg_pkgmanager = ("You can install cmake on {0} using "
                              "`sudo {1} install cmake`.")
            if sys.platform == "darwin":
                pkgmanagers = ('brew', 'port')
                for manager in pkgmanagers:
                    if find_executable(manager) is not None:
                        message = msg_pkgmanager.format('OSX', manager)
                        break
            elif sys.platform.startswith('linux'):
                try:
                    import distro
                except ImportError as err:
                    import pip
                    pip_exit = pip.main(['install', '-q', 'distro'])
                    if pip_exit > 0:
                        log.debug("Unable to install `distro` to identify "
                                  "the recommended command. Falling back "
                                  "to default error message.")
                        distro = err
                    else:
                        import distro
                if not isinstance(distro, ImportError):
                    distname = distro.id()
                    if distname in ('debian', 'ubuntu'):
                        message = msg_pkgmanager.format(
                            distname.title(), 'apt-get')
                    elif distname in ('fedora', 'centos', 'redhat'):
                        pkgmanagers = ("dnf", "yum")
                        for manager in pkgmanagers:
                            if find_executable(manager) is not None:
                                message = msg_pkgmanager.format(
                                    distname.title(), manager)
                                break
            raise DistutilsSetupError(
                "Cannot find cmake, ensure it is installed and in the path.\n"
                + message + "\n"
                "You can also specify its path with --cmake parameter.")

        platform_arch = platform.architecture()[0]
        log.info("Detected Python architecture: %s" % platform_arch)

        # make sure build artifacts are generated for the version of Python currently running
        cmake_extra_arch = []

        inc_dir = get_python_inc()
        lib_dir = get_config_var('LIBDIR')
        if (inc_dir != None):
            cmake_extra_arch += ['-DPYTHON_INCLUDE_DIR=' + inc_dir]
        if (lib_dir != None):
            cmake_extra_arch += ['-DCMAKE_LIBRARY_PATH=' + lib_dir]

        if sys.version_info >= (3, 0):
            cmake_extra_arch += ['-DPYTHON3=yes']

        log.info("Detected platform: %s" % sys.platform)
        if sys.platform == "darwin":
            # build on OS X

            # by default, cmake will choose the system python lib in /usr/lib
            # this checks the sysconfig and will correctly pick up a brewed python lib
            # e.g. in /usr/local/Cellar
            py_ver = get_python_version()
            # check: in some virtual environments the libpython has the form "libpython_#m.dylib
            py_lib = os.path.join(get_config_var('LIBDIR'),
                                  'libpython' + py_ver + '.dylib')
            if not os.path.isfile(py_lib):
                py_lib = os.path.join(get_config_var('LIBDIR'),
                                      'libpython' + py_ver + 'm.dylib')

            cmake_extra_arch += ['-DPYTHON_LIBRARY={lib}'.format(lib=py_lib)]

        if sys.platform == "win32":
            if platform_arch == '64bit' and not generator_set:
                cmake_extra_arch += get_msvc_win64_generator()

            # this imitates cmake in path resolution
            py_ver = get_python_version()
            for ext in [
                    py_ver.replace(".", "") + '.lib', py_ver + 'mu.lib',
                    py_ver + 'm.lib', py_ver + 'u.lib'
            ]:
                py_lib = os.path.abspath(
                    os.path.join(inc_dir, '../libs/', 'python' + ext))
                if os.path.exists(py_lib):
                    cmake_extra_arch += [
                        '-DPYTHON_LIBRARY={lib}'.format(lib=py_lib)
                    ]
                    break

        build_dir = os.path.join(script_dir, "./tools/python/build")
        if os.path.exists(build_dir):
            log.info('Removing build directory %s' % build_dir)
            rmtree(build_dir)

        try:
            os.makedirs(build_dir)
        except OSError:
            pass

        # cd build
        os.chdir(build_dir)
        log.info('Configuring cmake ...')
        cmake_cmd = [
            cmake_path,
            "..",
        ] + cmake_extra + cmake_extra_arch
        if run_process(cmake_cmd):
            raise DistutilsSetupError("cmake configuration failed!")

        log.info('Build using cmake ...')

        cmake_cmd = [
            cmake_path,
            "--build",
            ".",
            "--config",
            cmake_config,
            "--target",
            "install",
        ]

        if run_process(cmake_cmd):
            raise DistutilsSetupError("cmake build failed!")

        # cd back where setup awaits
        os.chdir(script_dir)
Пример #42
0
    def build_executable(self, exe):
        ListType, TupleType = type([]), type(())
        from distutils.dep_util import newer_group
        sources = exe.sources
        if sources is None or type(sources) not in (ListType, TupleType):
            raise DistutilsSetupError(
                ("in 'executables' option (executable '%s'), " +
                 "'sources' must be present and must be " +
                 "a list of source filenames") % exe.name)
        sources = list(sources)
        exe_filename = os.path.join(self.build_exe, exe.name)
        depends = sources + exe.depends
        if not (self.force or newer_group(depends, exe_filename, 'newer')):
            log.debug("skipping '%s' executable (up-to-date)", exe.name)
            return
        else:
            log.info("building '%s' executable", exe.name)

        # Next, compile the source code to object files.

        # XXX not honouring 'define_macros' or 'undef_macros' -- the
        # CCompiler API needs to change to accommodate this, and I
        # want to do one thing at a time!

        macros = exe.define_macros[:]
        for undef in exe.undef_macros:
            macros.append((undef, ))

        # Two possible sources for extra compiler arguments:
        #   - 'extra_compile_args' in Extension object
        #   - CFLAGS environment variable (not particularly
        #     elegant, but people seem to expect it and I
        #     guess it's useful)
        # The environment variable should take precedence, and
        # any sensible compiler will give precedence to later
        # command line args.  Hence we combine them in order:
        extra_args = exe.extra_compile_args or []
        extra_args = extra_args[:]

        objects = self.compiler.compile(sources,
                                        output_dir=self.build_temp,
                                        macros=macros,
                                        include_dirs=exe.include_dirs,
                                        debug=self.debug,
                                        extra_postargs=extra_args,
                                        depends=exe.depends)

        # XXX -- this is a Vile HACK!
        #
        # The setup.py script for Python on Unix needs to be able to
        # get this list so it can perform all the clean up needed to
        # avoid keeping object files around when cleaning out a failed
        # build of an extension module.  Since Distutils does not
        # track dependencies, we have to get rid of intermediates to
        # ensure all the intermediates will be properly re-built.
        #
        self._built_objects = objects[:]

        # XXX -- this is a Vile HACK!
        #
        # Remove msvcrXX.dll when building executables with MinGW
        #
        if self.compiler.compiler_type == 'mingw32':
            try:
                del self.compiler.dll_libraries[:]
            except:
                pass

        # Now link the object files together into a "shared object" --
        # of course, first we have to figure out all the other things
        # that go into the mix.
        if exe.extra_objects:
            objects.extend(exe.extra_objects)
        extra_args = exe.extra_link_args or []
        extra_args = extra_args[:]
        # Get special linker flags for building a executable with
        # bundled Python library, also fix location of needed
        # python.exp file on AIX
        ldshflag = sysconfig.get_config_var('LINKFORSHARED') or ''
        if sys.platform.startswith('aix'):
            python_lib = sysconfig.get_python_lib(standard_lib=1)
            python_exp = os.path.join(python_lib, 'config', 'python.exp')
            ldshflag = ldshflag.replace('Modules/python.exp', python_exp)
        extra_args.extend(ldshflag.split())
        # Detect target language, if not provided
        language = exe.language or self.compiler.detect_language(sources)
        self.compiler.link_executable(
            objects,
            exe_filename,
            output_dir=None,
            libraries=self.get_libraries(exe),
            library_dirs=exe.library_dirs,
            runtime_library_dirs=exe.runtime_library_dirs,
            extra_preargs=None,
            extra_postargs=extra_args,
            debug=self.debug,
            target_lang=language)
Пример #43
0
            for base in HKEYS:
                if read_values(base, r"%s\6.0" % self.__root) is not None:
                    self.warn("It seems you have Visual Studio 6 installed, "
                        "but the expected registry settings are not present.\n"
                        "You must at least run the Visual Studio GUI once "
                        "so that these entries are created.")
                    break
        return []

    def set_path_env_var(self, name):
        """Set environment variable 'name' to an MSVC path type value.

        This is equivalent to a SET command prior to execution of spawned
        commands.
        """

        if name == "lib":
            p = self.get_msvc_paths("library")
        else:
            p = self.get_msvc_paths(name)
        if p:
            os.environ[name] = ';'.join(p)


if get_build_version() >= 8.0:
    log.debug("Importing new compiler from distutils.msvc9compiler")
    OldMSVCCompiler = MSVCCompiler
    from distutils.msvc9compiler import MSVCCompiler
    # get_build_architecture not really relevant now we support cross-compile
    from distutils.msvc9compiler import MacroExpander
Пример #44
0
        if not self.initialized: self.initialize()
        (objects, output_dir) = self._fix_object_args(objects, output_dir)
        output_filename = \
            self.library_filename (output_libname, output_dir=output_dir)

        if self._need_link(objects, output_filename):
            lib_args = objects + ['/OUT:' + output_filename]
            if debug:
                pass  # XXX what goes here?
            try:
                self.spawn([self.lib] + lib_args)
            except DistutilsExecError, msg:
                raise LibError, msg

        else:
            log.debug("skipping %s (up-to-date)", output_filename)

    # create_static_lib ()

    def link(self,
             target_desc,
             objects,
             output_filename,
             output_dir=None,
             libraries=None,
             library_dirs=None,
             runtime_library_dirs=None,
             export_symbols=None,
             debug=0,
             extra_preargs=None,
             extra_postargs=None,
Пример #45
0
def find_vcvarsall(version):
    """Find the vcvarsall.bat file

    At first it tries to find the productdir of VS 2008 in the registry. If
    that fails it falls back to the VS90COMNTOOLS env var.
    """
    vsbase = VS_BASE % version
    try:
        productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
                                   "productdir")
    except KeyError:
        log.debug("Unable to find productdir in registry")
        productdir = None

    # trying Express edition
    if productdir is None:
        vsbase = VSEXPRESS_BASE % version
        try:
            productdir = Reg.get_value(r"%s\Setup\VC" % vsbase,
                                       "productdir")
        except KeyError:
            productdir = None
            log.debug("Unable to find productdir in registry")

    if not productdir or not os.path.isdir(productdir):
        toolskey = "VS%0.f0COMNTOOLS" % version
        toolsdir = os.environ.get(toolskey, None)

        if toolsdir and os.path.isdir(toolsdir):
            productdir = os.path.join(toolsdir, os.pardir, os.pardir, "VC")
            productdir = os.path.abspath(productdir)
            if not os.path.isdir(productdir):
                log.debug("%s is not a valid directory" % productdir)
                return None
        else:
            log.debug("Env var %s is not set or invalid" % toolskey)
    if not productdir:
        log.debug("No productdir found")
        return None
    vcvarsall = os.path.join(productdir, "vcvarsall.bat")
    if os.path.isfile(vcvarsall):
        return vcvarsall
    log.debug("Unable to find vcvarsall.bat")
    return None
Пример #46
0
def copy_file(src,
              dst,
              preserve_mode=1,
              preserve_times=1,
              update=0,
              link=None,
              verbose=0,
              dry_run=0):
    """Copy a file 'src' to 'dst'.  If 'dst' is a directory, then 'src' is
    copied there with the same name; otherwise, it must be a filename.  (If
    the file exists, it will be ruthlessly clobbered.)  If 'preserve_mode'
    is true (the default), the file's mode (type and permission bits, or
    whatever is analogous on the current platform) is copied.  If
    'preserve_times' is true (the default), the last-modified and
    last-access times are copied as well.  If 'update' is true, 'src' will
    only be copied if 'dst' does not exist, or if 'dst' does exist but is
    older than 'src'.

    'link' allows you to make hard links (os.link) or symbolic links
    (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
    None (the default), files are copied.  Don't set 'link' on systems that
    don't support it: 'copy_file()' doesn't check if hard or symbolic
    linking is available.

    Under Mac OS, uses the native file copy function in macostools; on
    other systems, uses '_copy_file_contents()' to copy file contents.

    Return a tuple (dest_name, copied): 'dest_name' is the actual name of
    the output file, and 'copied' is true if the file was copied (or would
    have been copied, if 'dry_run' true).
    """
    # XXX if the destination file already exists, we clobber it if
    # copying, but blow up if linking.  Hmmm.  And I don't know what
    # macostools.copyfile() does.  Should definitely be consistent, and
    # should probably blow up if destination exists and we would be
    # changing it (ie. it's not already a hard/soft link to src OR
    # (not update) and (src newer than dst).

    from distutils.dep_util import newer
    from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE

    if not os.path.isfile(src):
        raise DistutilsFileError, \
              "can't copy '%s': doesn't exist or not a regular file" % src

    if os.path.isdir(dst):
        dir = dst
        dst = os.path.join(dst, os.path.basename(src))
    else:
        dir = os.path.dirname(dst)

    if update and not newer(src, dst):
        log.debug("not copying %s (output up-to-date)", src)
        return dst, 0

    try:
        action = _copy_action[link]
    except KeyError:
        raise ValueError, \
              "invalid value '%s' for 'link' argument" % link
    if os.path.basename(dst) == os.path.basename(src):
        log.info("%s %s -> %s", action, src, dir)
    else:
        log.info("%s %s -> %s", action, src, dst)

    if dry_run:
        return (dst, 1)

    # On Mac OS, use the native file copy routine
    if os.name == 'mac':
        import macostools
        try:
            macostools.copy(src, dst, 0, preserve_times)
        except os.error, exc:
            raise DistutilsFileError, \
                  "could not copy '%s' to '%s': %s" % (src, dst, exc[-1])
Пример #47
0
    def link(self,
             target_desc,
             objects,
             output_filename,
             output_dir=None,
             libraries=None,
             library_dirs=None,
             runtime_library_dirs=None,
             export_symbols=None,
             debug=0,
             extra_preargs=None,
             extra_postargs=None,
             build_temp=None,
             target_lang=None):
        objects, output_dir = self._fix_object_args(objects, output_dir)
        libraries, library_dirs, runtime_library_dirs = self._fix_lib_args(
            libraries, library_dirs, runtime_library_dirs)
        if runtime_library_dirs:
            log.warn("I don't know what to do with 'runtime_library_dirs': %s",
                     str(runtime_library_dirs))
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)
        if self._need_link(objects, output_filename):
            if target_desc == CCompiler.EXECUTABLE:
                startup_obj = 'c0w32'
                if debug:
                    ld_args = self.ldflags_exe_debug[:]
                else:
                    ld_args = self.ldflags_exe[:]
            else:
                startup_obj = 'c0d32'
                if debug:
                    ld_args = self.ldflags_shared_debug[:]
                else:
                    ld_args = self.ldflags_shared[:]
            if export_symbols is None:
                def_file = ''
            else:
                head, tail = os.path.split(output_filename)
                modname, ext = os.path.splitext(tail)
                temp_dir = os.path.dirname(objects[0])
                def_file = os.path.join(temp_dir, '%s.def' % modname)
                contents = ['EXPORTS']
                for sym in export_symbols or []:
                    contents.append('  %s=_%s' % (sym, sym))

                self.execute(write_file, (def_file, contents),
                             'writing %s' % def_file)
            objects2 = map(os.path.normpath, objects)
            objects = [startup_obj]
            resources = []
            for file in objects2:
                base, ext = os.path.splitext(os.path.normcase(file))
                if ext == '.res':
                    resources.append(file)
                objects.append(file)

            for l in library_dirs:
                ld_args.append('/L%s' % os.path.normpath(l))

            ld_args.append('/L.')
            ld_args.extend(objects)
            ld_args.extend([',', output_filename])
            ld_args.append(',,')
            for lib in libraries:
                libfile = self.find_library_file(library_dirs, lib, debug)
                if libfile is None:
                    ld_args.append(lib)
                ld_args.append(libfile)

            ld_args.append('import32')
            ld_args.append('cw32mt')
            ld_args.extend([',', def_file])
            ld_args.append(',')
            ld_args.extend(resources)
            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)
            self.mkpath(os.path.dirname(output_filename))
            try:
                self.spawn([self.linker] + ld_args)
            except DistutilsExecError as msg:
                raise LinkError, msg

        else:
            log.debug('skipping %s (up-to-date)', output_filename)
        return
Пример #48
0
def run_prospector(base_dir, clear_ignore_patterns=False):
    """Run prospector and apply white/blacklists to the results"""
    orig_expand_default = optparse.HelpFormatter.expand_default

    sys.argv = ['fakename']
    sys.argv.extend(PROSPECTOR_OPTIONS)

    if PROSPECTOR_USE_LIBS:
        sys.argv.extend(["--uses", ",".join(PROSPECTOR_USE_LIBS)])

    # add/set REPO_BASE_DIR as positional path
    sys.argv.append(base_dir)
    log.debug("prospector commandline %s" % sys.argv)

    config = ProspectorConfig()
    # prospector will sometimes wrongly autodetect django
    config.libraries = []
    prospector = Prospector(config)

    # Set defaults for prospector:
    config.profile.doc_warnings = False
    config.profile.test_warnings = False
    config.profile.autodetect = True
    config.profile.member_warnings = False
    if clear_ignore_patterns:
        config.profile.ignore_patterns = [r'.^']
        config.ignores = []
    else:
        append_pattern = r'(^|/)\..+'
        config.profile.ignore_patterns.append(append_pattern)
        config.ignores.append(re.compile(append_pattern))

    # Enable pylint Python3 compatibility tests:
    config.profile.pylint['options']['enable'] = 'python3'
    log.debug("prospector argv = %s" % sys.argv)
    log.debug("prospector profile from config = %s" % vars(config.profile))

    prospector.execute()
    log.debug("prospector profile form prospector = %s" % vars(prospector.config.profile))

    blacklist = map(re.compile, PROSPECTOR_BLACKLIST)
    whitelist = map(re.compile, PROSPECTOR_WHITELIST)

    failures = []
    for msg in prospector.get_messages():
        # example msg.as_dict():
        #  {'source': 'pylint', 'message': 'Missing function docstring', 'code': 'missing-docstring',
        #   'location': {'function': 'TestHeaders.test_check_header.lgpl', 'path': u'headers.py',
        #                'line': 122, 'character': 8, 'module': 'headers'}}
        log.debug("prospector message %s" % msg.as_dict())

        if any([bool(reg.search(msg.code) or reg.search(msg.message)) for reg in blacklist]):
            continue

        if any([bool(reg.search(msg.code) or reg.search(msg.message)) for reg in whitelist]):
            failures.append(msg.as_dict())

    # There is some ugly monkeypatch code in pylint
    #     (or logilab if no recent enough pylint is installed)
    # Make sure the original is restored
    # (before any errors are reported; no need to put this in setUp/tearDown)
    optparse.HelpFormatter.expand_default = orig_expand_default

    return failures
Пример #49
0
    def run(self):
        if (sys.platform != "win32"
                and (self.distribution.has_ext_modules()
                     or self.distribution.has_c_libraries())):
            raise DistutilsPlatformError \
                  ("distribution contains extensions and/or C libraries; "
                   "must be compiled on a Windows 32 platform")

        if not self.skip_build:
            self.run_command('build')

        install = self.reinitialize_command('install', reinit_subcommands=1)
        install.root = self.bdist_dir
        install.skip_build = self.skip_build
        install.warn_dir = 0

        install_lib = self.reinitialize_command('install_lib')
        # we do not want to include pyc or pyo files
        install_lib.compile = 0
        install_lib.optimize = 0

        if self.distribution.has_ext_modules():
            # If we are building an installer for a Python version other
            # than the one we are currently running, then we need to ensure
            # our build_lib reflects the other Python version rather than ours.
            # Note that for target_version!=sys.version, we must have skipped the
            # build step, so there is no issue with enforcing the build of this
            # version.
            target_version = self.target_version
            if not target_version:
                assert self.skip_build, "Should have already checked this"
                target_version = sys.version[0:3]
            plat_specifier = ".%s-%s" % (get_platform(), target_version)
            build = self.get_finalized_command('build')
            build.build_lib = os.path.join(build.build_base,
                                           'lib' + plat_specifier)

        # Use a custom scheme for the zip-file, because we have to decide
        # at installation time which scheme to use.
        for key in ('purelib', 'platlib', 'headers', 'scripts', 'data'):
            value = string.upper(key)
            if key == 'headers':
                value = value + '/Include/$dist_name'
            setattr(install, 'install_' + key, value)

        log.info("installing to %s", self.bdist_dir)
        install.ensure_finalized()

        # avoid warning of 'install_lib' about installing
        # into a directory not in sys.path
        sys.path.insert(0, os.path.join(self.bdist_dir, 'PURELIB'))

        install.run()

        del sys.path[0]

        # And make an archive relative to the root of the
        # pseudo-installation tree.
        from tempfile import mktemp
        archive_basename = mktemp()
        fullname = self.distribution.get_fullname()
        arcname = self.make_archive(archive_basename,
                                    "zip",
                                    root_dir=self.bdist_dir)
        # create an exe containing the zip-file
        self.create_exe(arcname, fullname, self.bitmap)
        if self.distribution.has_ext_modules():
            pyversion = get_python_version()
        else:
            pyversion = 'any'
        self.distribution.dist_files.append(
            ('bdist_wininst', pyversion,
             self.get_installer_filename(fullname)))
        # remove the zip-file again
        log.debug("removing temporary file '%s'", arcname)
        os.remove(arcname)

        if not self.keep_temp:
            remove_tree(self.bdist_dir, dry_run=self.dry_run)
Пример #50
0
 def _run_yarn_command(self, cmd, env=None):
     log.debug(u'yarn path: ({0}))'.format(YARN_PATH))
     self._run_command(
         [YARN_PATH] + cmd, env=env
     )
Пример #51
0
    def link (self,
              target_desc,
              objects,
              output_filename,
              output_dir=None,
              libraries=None,
              library_dirs=None,
              runtime_library_dirs=None,
              export_symbols=None,
              debug=0,
              extra_preargs=None,
              extra_postargs=None,
              build_temp=None,
              target_lang=None):
        # First fixup.
        (objects, output_dir) = self._fix_object_args (objects, output_dir)
        (libraries, library_dirs, runtime_library_dirs) = \
            self._fix_lib_args (libraries, library_dirs, runtime_library_dirs)

        # First examine a couple of options for things that aren't implemented yet
        if not target_desc in (self.SHARED_LIBRARY, self.SHARED_OBJECT):
            raise DistutilsPlatformError, 'Can only make SHARED_LIBRARY or SHARED_OBJECT targets on the Mac'
        if runtime_library_dirs:
            raise DistutilsPlatformError, 'Runtime library dirs not implemented yet'
        if extra_preargs or extra_postargs:
            raise DistutilsPlatformError, 'Runtime library dirs not implemented yet'
        if len(export_symbols) != 1:
            raise DistutilsPlatformError, 'Need exactly one export symbol'
        # Next there are various things for which we need absolute pathnames.
        # This is because we (usually) create the project in a subdirectory of
        # where we are now, and keeping the paths relative is too much work right
        # now.
        sources = map(self._filename_to_abs, self.__sources)
        include_dirs = map(self._filename_to_abs, self.__include_dirs)
        if objects:
            objects = map(self._filename_to_abs, objects)
        else:
            objects = []
        if build_temp:
            build_temp = self._filename_to_abs(build_temp)
        else:
            build_temp = os.curdir()
        if output_dir:
            output_filename = os.path.join(output_dir, output_filename)
        # The output filename needs special handling: splitting it into dir and
        # filename part. Actually I'm not sure this is really needed, but it
        # can't hurt.
        output_filename = self._filename_to_abs(output_filename)
        output_dir, output_filename = os.path.split(output_filename)
        # Now we need the short names of a couple of things for putting them
        # into the project.
        if output_filename[-8:] == '.ppc.slb':
            basename = output_filename[:-8]
        elif output_filename[-11:] == '.carbon.slb':
            basename = output_filename[:-11]
        else:
            basename = os.path.strip(output_filename)[0]
        projectname = basename + '.mcp'
        targetname = basename
        xmlname = basename + '.xml'
        exportname = basename + '.mcp.exp'
        prefixname = 'mwerks_%s_config.h'%basename
        # Create the directories we need
        distutils.dir_util.mkpath(build_temp, dry_run=self.dry_run)
        distutils.dir_util.mkpath(output_dir, dry_run=self.dry_run)
        # And on to filling in the parameters for the project builder
        settings = {}
        settings['mac_exportname'] = exportname
        settings['mac_outputdir'] = output_dir
        settings['mac_dllname'] = output_filename
        settings['mac_targetname'] = targetname
        settings['sysprefix'] = sys.prefix
        settings['mac_sysprefixtype'] = 'Absolute'
        sourcefilenames = []
        sourcefiledirs = []
        for filename in sources + objects:
            dirname, filename = os.path.split(filename)
            sourcefilenames.append(filename)
            if not dirname in sourcefiledirs:
                sourcefiledirs.append(dirname)
        settings['sources'] = sourcefilenames
        settings['libraries'] = libraries
        settings['extrasearchdirs'] = sourcefiledirs + include_dirs + library_dirs
        if self.dry_run:
            print 'CALLING LINKER IN', os.getcwd()
            for key, value in settings.items():
                print '%20.20s %s'%(key, value)
            return
        # Build the export file
        exportfilename = os.path.join(build_temp, exportname)
        log.debug("\tCreate export file %s", exportfilename)
        fp = open(exportfilename, 'w')
        fp.write('%s\n'%export_symbols[0])
        fp.close()
        # Generate the prefix file, if needed, and put it in the settings
        if self.__macros:
            prefixfilename = os.path.join(os.getcwd(), os.path.join(build_temp, prefixname))
            fp = open(prefixfilename, 'w')
            fp.write('#include "mwerks_shcarbon_config.h"\n')
            for name, value in self.__macros:
                if value is None:
                    fp.write('#define %s\n'%name)
                else:
                    fp.write('#define %s %s\n'%(name, value))
            fp.close()
            settings['prefixname'] = prefixname

        # Build the XML file. We need the full pathname (only lateron, really)
        # because we pass this pathname to CodeWarrior in an AppleEvent, and CW
        # doesn't have a clue about our working directory.
        xmlfilename = os.path.join(os.getcwd(), os.path.join(build_temp, xmlname))
        log.debug("\tCreate XML file %s", xmlfilename)
        xmlbuilder = mkcwproject.cwxmlgen.ProjectBuilder(settings)
        xmlbuilder.generate()
        xmldata = settings['tmp_projectxmldata']
        fp = open(xmlfilename, 'w')
        fp.write(xmldata)
        fp.close()
        # Generate the project. Again a full pathname.
        projectfilename = os.path.join(os.getcwd(), os.path.join(build_temp, projectname))
        log.debug('\tCreate project file %s', projectfilename)
        mkcwproject.makeproject(xmlfilename, projectfilename)
        # And build it
        log.debug('\tBuild project')
        mkcwproject.buildproject(projectfilename)
Пример #52
0
    def run(self):
        if has_system_lib():
            log.info('Using system library')
            return

        build_temp = os.path.abspath(self.build_temp)

        try:
            os.makedirs(build_temp)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        if not os.path.exists(absolute('libsecp256k1')):
            # library needs to be downloaded
            self.get_source_files()

        if not os.path.exists(absolute('libsecp256k1/configure')):
            # configure script hasn't been generated yet
            autogen = absolute('libsecp256k1/autogen.sh')
            os.chmod(absolute(autogen), 0o755)
            subprocess.check_call([autogen], cwd=absolute('libsecp256k1'))

        for filename in [
                'libsecp256k1/configure',
                'libsecp256k1/build-aux/compile',
                'libsecp256k1/build-aux/config.guess',
                'libsecp256k1/build-aux/config.sub',
                'libsecp256k1/build-aux/depcomp',
                'libsecp256k1/build-aux/install-sh',
                'libsecp256k1/build-aux/missing',
                'libsecp256k1/build-aux/test-driver',
        ]:
            try:
                os.chmod(absolute(filename), 0o755)
            except OSError as e:
                # some of these files might not exist depending on autoconf version
                if e.errno != errno.ENOENT:
                    # If the error isn't 'No such file or directory' something
                    # else is wrong and we want to know about it
                    raise

        cmd = [
            absolute('libsecp256k1/configure'),
            '--disable-shared',
            '--enable-static',
            '--disable-dependency-tracking',
            '--with-pic',
            '--enable-module-recovery',
            '--disable-jni',
            '--prefix',
            os.path.abspath(self.build_clib),
            '--enable-experimental',
            '--enable-module-ecdh',
            '--enable-benchmark=no',
            #  '--enable-endomorphism',
        ]

        log.debug('Running configure: {}'.format(' '.join(cmd)))
        subprocess.check_call(cmd, cwd=build_temp)

        subprocess.check_call([MAKE], cwd=build_temp)
        subprocess.check_call([MAKE, 'install'], cwd=build_temp)

        self.build_flags['include_dirs'].extend(
            build_flags('libsecp256k1', 'I', build_temp))
        self.build_flags['library_dirs'].extend(
            build_flags('libsecp256k1', 'L', build_temp))
        if not has_system_lib():
            self.build_flags['define'].append(('CFFI_ENABLE_RECOVERY', None))
        else:
            pass
Пример #53
0
    def finalize_options(self):
        # This requires some explanation.  Basically what we want to do
        # here is to control if we want to build in-place or into the
        # build-lib folder.  Traditionally this is set by the `inplace`
        # command line flag for build_ext.  However as we are a subcommand
        # we need to grab this information from elsewhere.
        #
        # An in-place build puts the files generated into the source
        # folder, a regular build puts the files into the build-lib
        # folder.
        #
        # The following situations we need to cover:
        #
        #   command                         default in-place
        #   setup.py build_js               0
        #   setup.py build_ext              value of in-place for build_ext
        #   setup.py build_ext --inplace    1
        #   pip install --editable .        1
        #   setup.py install                0
        #   setup.py sdist                  0
        #   setup.py bdist_wheel            0
        #
        # The way this is achieved is that build_js is invoked by two
        # subcommands: bdist_ext (which is in our case always executed
        # due to a custom distribution) or sdist.
        #
        # Note: at one point install was an in-place build but it's not
        # quite sure why.  In case a version of install breaks again:
        # installations via pip from git URLs definitely require the
        # in-place flag to be disabled.  So we might need to detect
        # that separately.
        #
        # To find the default value of the inplace flag we inspect the
        # sdist and build_ext commands.
        sdist = self.distribution.get_command_obj('sdist')
        build_ext = self.get_finalized_command('build_ext')

        # If we are not decided on in-place we are inplace if either
        # build_ext is inplace or we are invoked through the install
        # command (easiest check is to see if it's finalized).
        if self.inplace is None:
            self.inplace = (build_ext.inplace or sdist.finalized) and 1 or 0

        # If we're coming from sdist, clear the hell out of the dist
        # folder first.
        if sdist.finalized:
            for path in self.get_dist_paths():
                try:
                    shutil.rmtree(path)
                except (OSError, IOError):
                    pass

        # In place means build_lib is src.  We also log this.
        if self.inplace:
            log.debug('in-place js building enabled')
            self.build_lib = 'src'
        # Otherwise we fetch build_lib from the build command.
        else:
            self.set_undefined_options('build', ('build_lib', 'build_lib'))
            log.debug('regular js build: build path is %s' % self.build_lib)

        if self.work_path is None:
            self.work_path = self.get_root_path()
Пример #54
0
    def link(self,
             target_desc,
             objects,
             output_filename,
             output_dir=None,
             libraries=None,
             library_dirs=None,
             runtime_library_dirs=None,
             export_symbols=None,
             debug=0,
             extra_preargs=None,
             extra_postargs=None,
             build_temp=None,
             target_lang=None):
        if not self.initialized:
            self.initialize()
        objects, output_dir = self._fix_object_args(objects, output_dir)
        fixed_args = self._fix_lib_args(libraries, library_dirs,
                                        runtime_library_dirs)
        libraries, library_dirs, runtime_library_dirs = fixed_args
        if runtime_library_dirs:
            self.warn("I don't know what to do with 'runtime_library_dirs': " +
                      str(runtime_library_dirs))
        lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs,
                                   libraries)
        if output_dir is not None:
            output_filename = os.path.join(output_dir, output_filename)
        if self._need_link(objects, output_filename):
            if target_desc == CCompiler.EXECUTABLE:
                if debug:
                    ldflags = self.ldflags_shared_debug[1:]
                else:
                    ldflags = self.ldflags_shared[1:]
            elif debug:
                ldflags = self.ldflags_shared_debug
            else:
                ldflags = self.ldflags_shared
            export_opts = []
            for sym in export_symbols or []:
                export_opts.append('/EXPORT:' + sym)

            ld_args = ldflags + lib_opts + export_opts + objects + [
                '/OUT:' + output_filename
            ]
            build_temp = os.path.dirname(objects[0])
            if export_symbols is not None:
                dll_name, dll_ext = os.path.splitext(
                    os.path.basename(output_filename))
                implib_file = os.path.join(build_temp,
                                           self.library_filename(dll_name))
                ld_args.append('/IMPLIB:' + implib_file)
            self.manifest_setup_ldargs(output_filename, build_temp, ld_args)
            if extra_preargs:
                ld_args[:0] = extra_preargs
            if extra_postargs:
                ld_args.extend(extra_postargs)
            self.mkpath(os.path.dirname(output_filename))
            try:
                self.spawn([self.linker] + ld_args)
            except DistutilsExecError as msg:
                raise LinkError(msg)

            mfinfo = self.manifest_get_embed_info(target_desc, ld_args)
            if mfinfo is not None:
                mffilename, mfid = mfinfo
                out_arg = '-outputresource:%s;%s' % (output_filename, mfid)
                try:
                    self.spawn([
                        'mt.exe', '-nologo', '-manifest', mffilename, out_arg
                    ])
                except DistutilsExecError as msg:
                    raise LinkError(msg)

        else:
            log.debug('skipping %s (up-to-date)', output_filename)
        return
Пример #55
0
def copy_file(src,
              dst,
              preserve_mode=1,
              preserve_times=1,
              update=0,
              link=None,
              verbose=1,
              dry_run=0):
    """Copy a file 'src' to 'dst'.

    If 'dst' is a directory, then 'src' is copied there with the same name;
    otherwise, it must be a filename.  (If the file exists, it will be
    ruthlessly clobbered.)  If 'preserve_mode' is true (the default),
    the file's mode (type and permission bits, or whatever is analogous on
    the current platform) is copied.  If 'preserve_times' is true (the
    default), the last-modified and last-access times are copied as well.
    If 'update' is true, 'src' will only be copied if 'dst' does not exist,
    or if 'dst' does exist but is older than 'src'.

    'link' allows you to make hard links (os.link) or symbolic links
    (os.symlink) instead of copying: set it to "hard" or "sym"; if it is
    None (the default), files are copied.  Don't set 'link' on systems that
    don't support it: 'copy_file()' doesn't check if hard or symbolic
    linking is available.

    Under Mac OS, uses the native file copy function in macostools; on
    other systems, uses '_copy_file_contents()' to copy file contents.

    Return a tuple (dest_name, copied): 'dest_name' is the actual name of
    the output file, and 'copied' is true if the file was copied (or would
    have been copied, if 'dry_run' true).
    """
    # XXX if the destination file already exists, we clobber it if
    # copying, but blow up if linking.  Hmmm.  And I don't know what
    # macostools.copyfile() does.  Should definitely be consistent, and
    # should probably blow up if destination exists and we would be
    # changing it (ie. it's not already a hard/soft link to src OR
    # (not update) and (src newer than dst).

    from distutils.dep_util import newer
    from stat import ST_ATIME, ST_MTIME, ST_MODE, S_IMODE

    if not os.path.isfile(src):
        raise DistutilsFileError(
            "can't copy '%s': doesn't exist or not a regular file" % src)

    if os.path.isdir(dst):
        dir = dst
        dst = os.path.join(dst, os.path.basename(src))
    else:
        dir = os.path.dirname(dst)

    if update and not newer(src, dst):
        if verbose >= 1:
            log.debug("not copying %s (output up-to-date)", src)
        return dst, 0

    try:
        action = _copy_action[link]
    except KeyError:
        raise ValueError("invalid value '%s' for 'link' argument" % link)

    if verbose >= 1:
        if os.path.basename(dst) == os.path.basename(src):
            log.info("%s %s -> %s", action, src, dir)
        else:
            log.info("%s %s -> %s", action, src, dst)

    if dry_run:
        return (dst, 1)

    # If linking (hard or symbolic), use the appropriate system call
    # (Unix only, of course, but that's the caller's responsibility)
    if link == 'hard':
        if not (os.path.exists(dst) and os.path.samefile(src, dst)):
            os.link(src, dst)
    elif link == 'sym':
        if not (os.path.exists(dst) and os.path.samefile(src, dst)):
            os.symlink(src, dst)

    # Otherwise (non-Mac, not linking), copy the file contents and
    # (optionally) copy the times and mode.
    else:
        _copy_file_contents(src, dst)
        if preserve_mode or preserve_times:
            st = os.stat(src)

            # According to David Ascher <*****@*****.**>, utime() should be done
            # before chmod() (at least under NT).
            if preserve_times:
                os.utime(dst, (st[ST_ATIME], st[ST_MTIME]))
            if preserve_mode and hasattr(os, 'chmod'):
                os.chmod(dst, S_IMODE(st[ST_MODE]))

    return (dst, 1)
Пример #56
0
from sage_setup.optional_extension import is_package_installed_and_updated

optional_packages_with_extensions = ['mcqd', 'bliss', 'tdlib', 'primecount',
                                     'coxeter3', 'fes', 'sirocco', 'meataxe']

distributions += ['sage-{}'.format(pkg)
                  for pkg in optional_packages_with_extensions
                  if is_package_installed_and_updated(pkg)]

log.warn('distributions = {0}'.format(distributions))

from sage_setup.find import find_python_sources
python_packages, python_modules, cython_modules = find_python_sources(
    SAGE_SRC, ['sage', 'sage_setup'], distributions=distributions)

log.debug('python_packages = {0}'.format(python_packages))

print("Discovered Python/Cython sources, time: %.2f seconds." % (time.time() - t))


from sage_setup.command.sage_install import sage_install

#########################################################
### Distutils
#########################################################

code = setup(name = 'sage',
      version     =  SAGE_VERSION,
      description = 'Sage: Open Source Mathematics Software',
      license     = 'GNU Public License (GPL)',
      author      = 'William Stein et al.',
Пример #57
0
def build_extension(self, ext):
    """Modified version of build_extension method from distutils.
       Can handle compiler args for different files"""

    sources = ext.sources
    if sources is None or not isinstance(sources, (list, tuple)):
        raise DistutilsSetupError("in 'ext_modules' option (extension '%s'), "
                                  "'sources' must be present and must be "
                                  "a list of source filenames" % ext.name)

    sources = list(sources)
    ext_path = self.get_ext_fullpath(ext.name)
    depends = sources + ext.depends
    if not (self.force or newer_group(depends, ext_path, 'newer')):
        log.debug("skipping '%s' extension (up-to-date)", ext.name)
        return
    else:
        log.info("building '%s' extension", ext.name)

    sources = self.swig_sources(sources, ext)

    extra_args = ext.extra_compile_args or []
    extra_c_args = getattr(ext, "extra_compile_c_args", [])
    extra_cpp_args = getattr(ext, "extra_compile_cpp_args", [])
    extra_objc_args = getattr(ext, "extra_compile_objc_args", [])
    macros = ext.define_macros[:]
    for undef in ext.undef_macros:
        macros.append((undef, ))

    c_sources, cpp_sources, objc_sources, other_sources = filter_sources(
        sources)

    def _compile(src, args):
        return self.compiler.compile(src,
                                     output_dir=self.build_temp,
                                     macros=macros,
                                     include_dirs=ext.include_dirs,
                                     debug=self.debug,
                                     extra_postargs=extra_args + args,
                                     depends=ext.depends)

    objects = []
    objects += _compile(c_sources, extra_c_args)
    objects += _compile(cpp_sources, extra_cpp_args)
    objects += _compile(objc_sources, extra_objc_args)
    objects += _compile(other_sources, [])

    self._built_objects = objects[:]
    if ext.extra_objects:
        objects.extend(ext.extra_objects)

    extra_args = ext.extra_link_args or []

    language = ext.language or self.compiler.detect_language(sources)
    self.compiler.link_shared_object(
        objects,
        ext_path,
        libraries=self.get_libraries(ext),
        library_dirs=ext.library_dirs,
        runtime_library_dirs=ext.runtime_library_dirs,
        extra_postargs=extra_args,
        export_symbols=self.get_export_symbols(ext),
        debug=self.debug,
        build_temp=self.build_temp,
        target_lang=language)