setup.py

from __future__ import print_function

from distutils.ccompiler import new_compiler as _new_compiler, LinkError, CompileError
from distutils.command.clean import clean, log
from distutils.core import Command
from distutils.dir_util import remove_tree
from distutils.errors import DistutilsExecError
from distutils.msvccompiler import MSVCCompiler
from setuptools import setup, find_packages, Extension, Distribution
from setuptools.command.build_ext import build_ext
from subprocess import Popen, PIPE
import errno
import itertools
import json
import os
import platform
import re
import sys

try:
    # This depends on _winreg, which is not availible on not-Windows.
    from distutils.msvc9compiler import MSVCCompiler as MSVC9Compiler
except ImportError:
    MSVC9Compiler = None

try:
    from distutils._msvccompiler import MSVCCompiler as MSVC14Compiler
except ImportError:
    MSVC14Compiler = None


msvc_compiler_classes = tuple([cls for cls in (MSVCCompiler, MSVC9Compiler,
                                               MSVC14Compiler) if cls is not None])

try:
    from Cython.Build import cythonize
except ImportError:
    # We don't need Cython all the time; just for building from original source.
    cythonize = None


is_py3 = sys.version_info[0] >= 3


# We will embed this metadata into the package so it can be recalled for debugging.
version = '0.4.1.dev0'
try:
    git_commit, _ = Popen(['git', 'describe', '--tags'], stdout=PIPE, stderr=PIPE).communicate()
except OSError:
    git_commit = None
else:
    git_commit = git_commit.strip()


def get_library_config(name):
    """Get distutils-compatible extension extras for the given library.

    This requires ``pkg-config``.

    """
    try:
        proc = Popen(['pkg-config', '--cflags', '--libs', name], stdout=PIPE, stderr=PIPE)
    except OSError:
        print('pkg-config is required for building PyAV')
        exit(1)

    raw_config, err = proc.communicate()
    if proc.wait():
        return
    config = {}
    for chunk in raw_config.decode('utf8').strip().split():
        if chunk.startswith('-I'):
            config.setdefault('include_dirs', []).append(chunk[2:])
        elif chunk.startswith('-L'):
            config.setdefault('library_dirs', []).append(chunk[2:])
        elif chunk.startswith('-l'):
            config.setdefault('libraries', []).append(chunk[2:])
        elif chunk.startswith('-D'):
            name = chunk[2:].split('=')[0]
            config.setdefault('define_macros', []).append((name, None))

    return config


def update_extend(dst, src):
    """Update the `dst` with the `src`, extending values where lists.

    Primiarily useful for integrating results from `get_library_config`.

    """
    for k, v in src.items():
        existing = dst.setdefault(k, [])
        for x in v:
            if x not in existing:
                existing.append(x)


def unique_extend(a, *args):
    a[:] = list(set().union(a, *args))


def is_msvc(cc=None):
    cc = _new_compiler() if cc is None else cc
    return isinstance(cc, msvc_compiler_classes)

# Obtain the ffmpeg dir from the "--ffmpeg-dir=<dir>" argument
FFMPEG_DIR = None
for i, arg in enumerate(sys.argv):
    if arg.startswith('--ffmpeg-dir='):
        FFMPEG_DIR = arg.split('=')[1]
        break

if FFMPEG_DIR is not None:
    # delete the --ffmpeg-dir arg so that distutils does not see it
    del sys.argv[i]
    if not os.path.isdir(FFMPEG_DIR):
        print('The specified ffmpeg directory does not exist')
        exit(1)
else:
    # Check the environment variable FFMPEG_DIR
    FFMPEG_DIR = os.environ.get('FFMPEG_DIR')
    if FFMPEG_DIR is not None:
        if not os.path.isdir(FFMPEG_DIR):
            FFMPEG_DIR = None

if FFMPEG_DIR is not None:
    ffmpeg_lib = os.path.join(FFMPEG_DIR, 'lib')
    ffmpeg_include = os.path.join(FFMPEG_DIR, 'include')
    if os.path.exists(ffmpeg_lib):
        ffmpeg_lib = [ffmpeg_lib]
    else:
        ffmpeg_lib = [FFMPEG_DIR]
    if os.path.exists(ffmpeg_include):
        ffmpeg_include = [ffmpeg_include]
    else:
        ffmpeg_include = [FFMPEG_DIR]
else:
    ffmpeg_lib = []
    ffmpeg_include = []


# The "extras" to be supplied to every one of our modules.
# This is expanded heavily by the `config` command.
extension_extra = {
    'include_dirs': ['include'] + ffmpeg_include,  # The first are PyAV's includes.
    'libraries'   : [],
    'library_dirs': ffmpeg_lib,
}

# The macros which describe what functions and structure members we have
# from the underlying libraries. This is expanded heavily by `reflect` command.
config_macros = {
    "PYAV_VERSION": version,
    "PYAV_VERSION_STR": '"%s"' % version,
    "PYAV_COMMIT_STR": '"%s"' % (git_commit or 'unknown-commit'),
}


def dump_config():
    """Print out all the config information we have so far (for debugging)."""
    print('PyAV:', version, git_commit or '(unknown commit)')
    print('Python:', sys.version.encode('unicode_escape' if is_py3 else 'string-escape'))
    print('platform:', platform.platform())
    print('extension_extra:')
    for k, vs in extension_extra.items():
        print('\t%s: %s' % (k, [x.encode('utf8') for x in vs]))
    print('config_macros:')
    for x in sorted(config_macros.items()):
        print('\t%s=%s' % x)


if os.name == 'nt':


    if is_msvc():
        config_macros['inline'] = '__inline'
    # Since we're shipping a self contained unit on windows, we need to mark
    # the package as such. On other systems, let it be universal.
    class BinaryDistribution(Distribution):
        def is_pure(self):
            return False
    distclass = BinaryDistribution

else:

    distclass = Distribution


# Monkey-patch for CCompiler to be silent.
def _CCompiler_spawn_silent(cmd, dry_run=None):
    """Spawn a process, and eat the stdio."""
    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
    out, err = proc.communicate()
    if proc.returncode:
        raise DistutilsExecError(err)

def new_compiler(*args, **kwargs):
    """Create a C compiler.

    :param bool silent: Eat all stdio? Defaults to ``True``.

    All other arguments passed to ``distutils.ccompiler.new_compiler``.

    """
    cc = _new_compiler(*args, **kwargs)
    make_silent = True
    # If MSVC10, initialize the compiler here and add /MANIFEST to linker flags.
    # See Python issue 4431 (https://bugs.python.org/issue4431)
    if is_msvc(cc):
        from distutils.msvc9compiler import get_build_version
        if get_build_version() == 10:
            cc.initialize()
            for ldflags in [cc.ldflags_shared, cc.ldflags_shared_debug]:
                unique_extend(ldflags, ['/MANIFEST'])
        # If MSVC14, do not silence. As msvc14 requires some custom
        # steps before the process is spawned, we can't monkey-patch this.
        elif get_build_version() == 14:
            make_silent = False
    # monkey-patch compiler to suppress stdout and stderr.
    if make_silent and kwargs.pop('silent', True):
        cc.spawn = _CCompiler_spawn_silent
    return cc


def compile_check(code, name, includes=None, include_dirs=None, libraries=None,
                  library_dirs=None, link=True, compiler=None):
    """Check that we can compile and link the given source.

    Caches results; delete the ``build`` directory to reset.

    Writes source (``*.c``), builds (``*.o``), executables (``*.out``),
    and cached results (``*.json``) in ``build/temp.$platform/reflection``.

    """
    exec_path = name + '.out'
    source_path = name + '.c'
    result_path = name + '.json'

    if os.path.exists(result_path):
        try:
            return json.load(open(result_path))
        except ValueError:
            pass

    cc = new_compiler(compiler=compiler)

    with open(source_path, 'w') as fh:
        if is_msvc(cc):
            fh.write("#define inline __inline\n")
        for include in includes or ():
            fh.write('#include "%s"\n' % include)
        fh.write('main(int argc, char **argv)\n{ %s; }\n' % code)


    try:
        objects = cc.compile([source_path], include_dirs=include_dirs)
        if link:
            cc.link_executable(objects, exec_path, libraries=libraries, library_dirs=library_dirs)
    except (CompileError, LinkError, TypeError):
        res = False
    else:
        res = True

    with open(result_path, 'w') as fh:
        fh.write(json.dumps(res))

    return res


# Monkey-patch Cython to not overwrite embedded signatures.
if cythonize:

    from Cython.Compiler.AutoDocTransforms import EmbedSignature

    old_embed_signature = EmbedSignature._embed_signature
    def new_embed_signature(self, sig, doc):

        # Strip any `self` parameters from the front.
        sig = re.sub(r'\(self(,\s+)?', '(', sig)

        # If they both start with the same signature; skip it.
        if sig and doc:
            new_name = sig.split('(')[0].strip()
            old_name = doc.split('(')[0].strip()
            if new_name == old_name:
                return doc
            if new_name.endswith('.' + old_name):
                return doc

        return old_embed_signature(self, sig, doc)

    EmbedSignature._embed_signature = new_embed_signature


# Construct the modules that we find in the "av" directory.
ext_modules = []
for dirname, dirnames, filenames in os.walk('av'):
    for filename in filenames:

        # We are looing for Cython sources.
        if filename.startswith('.') or os.path.splitext(filename)[1] != '.pyx':
            continue

        pyx_path = os.path.join(dirname, filename)
        base = os.path.splitext(pyx_path)[0]

        # Need to be a little careful because Windows will accept / or \
        # (where os.sep will be \ on Windows).
        mod_name = base.replace('/', '.').replace(os.sep, '.')

        c_path = os.path.join('src', base + '.c')

        # We go with the C sources if Cython is not installed, and fail if
        # those also don't exist. We can't `cythonize` here though, since the
        # `pyav/include.h` must be generated (by `build_ext`) first.
        if not cythonize and not os.path.exists(c_path):
            print('Cython is required to build PyAV from raw sources.')
            print('Please `pip install Cython`.')
            exit(3)
        ext_modules.append(Extension(
            mod_name,
            sources=[c_path if not cythonize else pyx_path],
        ))


class ConfigCommand(Command):

    user_options = [
        ('no-pkg-config', None,
         "do not use pkg-config to configure dependencies"),
        ('compiler=', 'c',
         "specify the compiler type"), ]

    boolean_options = ['no-pkg-config']

    def initialize_options(self):
        self.compiler = None
        self.no_pkg_config = None

    def finalize_options(self):
        self.set_undefined_options('build',
            ('compiler', 'compiler'),)
        self.set_undefined_options('build_ext',
            ('no_pkg_config', 'no_pkg_config'),)

    def run(self):

        for name in 'libswresample', 'libavresample':
            # We will look for these in a moment.
            config_macros['PYAV_HAVE_' + name.upper()] = 0

        if is_msvc(new_compiler(compiler=self.compiler)):
            # Assume we have to disable /OPT:REF for MSVC with ffmpeg
            config = {
                'extra_link_args': ['/OPT:NOREF'],
            }
            update_extend(extension_extra, config)

        # Check if we're using pkg-config or not
        if self.no_pkg_config:
            # Simply assume we have everything we need!
            config = {
                'libraries':    ['avformat', 'avcodec', 'avdevice', 'avutil', 'avfilter',
                                 'swscale'],
                'library_dirs': [],
                'include_dirs': []
            }
            config['libraries'].append('swresample')
            config_macros['PYAV_HAVE_LIBSWRESAMPLE'] = 1
            update_extend(extension_extra, config)
            for ext in self.distribution.ext_modules:
                for key, value in extension_extra.items():
                    setattr(ext, key, value)
            return

        # We're using pkg-config:
        errors = []

        # Get the config for the libraries that we require.
        for name in 'libavformat', 'libavcodec', 'libavdevice', 'libavutil', 'libavfilter', 'libswscale':
            config = get_library_config(name)
            if config:
                update_extend(extension_extra, config)
                # We don't need macros for these, since they all must exist.
            else:
                errors.append('Could not find ' + name + ' with pkg-config.')

        # Get the config for either swresample OR avresample.
        for name in 'libswresample', 'libavresample':
            config = get_library_config(name)
            if config:
                update_extend(extension_extra, config)
                config_macros['PYAV_HAVE_' + name.upper()] = 1
                break
        else:
            errors.append('Could not find either libswresample or libavresample with pkg-config.')

        # Don't continue if we have errors.
        # TODO: Warn Ubuntu 12 users that they can't satisfy requirements with the
        # default package sources.
        if errors:
            print('\n'.join(errors))
            exit(1)

        # Normalize the extras.
        extension_extra.update(
            dict((k, sorted(set(v))) for k, v in extension_extra.items())
        )

        for ext in self.distribution.ext_modules:
            for key, value in extension_extra.items():
                setattr(ext, key, value)


class ReflectCommand(Command):

    sep_by = " (separated by '%s')" % os.pathsep
    user_options = [
        ('build-temp=', 't', "directory for temporary files (build by-products)"),
        ('include-dirs=', 'I', "list of directories to search for header files" + sep_by),
        ('libraries=', 'l', "external C libraries to link with"),
        ('library-dirs=', 'L', "directories to search for external C libraries" + sep_by),
        ('no-pkg-config', None, "do not use pkg-config to configure dependencies"),
        ('compiler=', 'c', "specify the compiler type"),
    ]

    boolean_options = ['no-pkg-config']

    def initialize_options(self):
        self.compiler = None
        self.build_temp = None
        self.include_dirs = None
        self.libraries = None
        self.library_dirs = None
        self.no_pkg_config = None

    def finalize_options(self):
        self.set_undefined_options('build',
            ('build_temp', 'build_temp'),
            ('compiler', 'compiler'),
        )
        self.set_undefined_options('build_ext',
            ('include_dirs', 'include_dirs'),
            ('libraries', 'libraries'),
            ('library_dirs', 'library_dirs'),
            ('no_pkg_config', 'no_pkg_config'),
        )
        # Need to do this ourself, since no inheritance from build_ext:
        try:
            str_base = basestring
        except NameError:
            str_base = str
        if isinstance(self.include_dirs, str_base):
            self.include_dirs = self.include_dirs.split(os.pathsep)
        if isinstance(self.library_dirs, str_base):
            self.library_dirs = str.split(self.library_dirs, os.pathsep)

    def run(self):

        # Propagate options
        obj = self.distribution.get_command_obj('config')
        obj.no_pkg_config = self.no_pkg_config
        obj.compiler = self.compiler
        self.run_command('config')

        tmp_dir = os.path.join(self.build_temp, 'reflection')
        try:
            os.makedirs(tmp_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        results = {}

        reflection_includes = [
            'libavcodec/avcodec.h',
            'libavformat/avformat.h',
            'libavutil/avutil.h',
            'libavutil/opt.h',
        ]

        config = extension_extra.copy()
        config['include_dirs'] += self.include_dirs
        config['libraries'] += self.libraries
        config['library_dirs'] += self.library_dirs

        # Check for some specific functions.
        for func_name in (

            'avformat_open_input', # Canary that should exist.
            'pyav_function_should_not_exist', # Canary that should not exist.

            # This we actually care about:
            'av_calloc',
            'av_frame_get_best_effort_timestamp',
            'avformat_alloc_output_context2',
            'avformat_close_input',
            'avcodec_send_packet',

        ):
            print("looking for %s... " % func_name, end='')
            results[func_name] = compile_check(
                name=os.path.join(tmp_dir, func_name),
                code='%s()' % func_name,
                libraries=config['libraries'],
                library_dirs=config['library_dirs'],
                compiler=self.compiler,
            )
            print('found' if results[func_name] else 'missing')

        # Check for some enum values.
        for enum_name in (
            'AV_OPT_TYPE_INT', # Canary that should exist.
            'PYAV_ENUM_SHOULD_NOT_EXIST', # Canary that should not exist.

            # What we actually care about.
            'AV_OPT_TYPE_BOOL',
        ):
            print("looking for %s..." % enum_name, end='')
            results[enum_name] = compile_check(
                name=os.path.join(tmp_dir, enum_name),
                code='int x = %s' % enum_name,
                includes=reflection_includes,
                include_dirs=config['include_dirs'],
                link=False,
                compiler=self.compiler,
            )
            print("found" if results[enum_name] else "missing")

        for struct_name, member_name in (

            ('AVStream', 'index'), # Canary that should exist
            ('PyAV', 'struct_should_not_exist'), # Canary that should not exist.

            # Things we actually care about:
            ('AVFrame', 'mb_type'),

        ):
            name = '%s.%s' % (struct_name, member_name)
            print("looking for %s... " % name, end='')
            results[name] = compile_check(
                name=os.path.join(tmp_dir, name),
                code='struct %s x; x.%s;' % (struct_name, member_name),
                includes=reflection_includes,
                include_dirs=config['include_dirs'],
                link=False,
                compiler=self.compiler,
            )
            print('found' if results[name] else 'missing')

        canaries = {
            'pyav_function_should_not_exist': False,
            'PyAV.struct_should_not_exist': False,
            'AV_OPT_TYPE_INT': True,
            'PYAV_ENUM_SHOULD_NOT_EXIST': False,
            'avformat_open_input': True,
            'AVStream.index': True,
        }

        # Create macros for the things that we found.
        # There is potential for naming collisions between functions and
        # structure members, but until we actually have one, we won't
        # worry about it.
        for name, value in results.items():
            if name in canaries:
                continue
            config_macros['PYAV_HAVE_%s' % name.upper().replace('.', '__')] = 1 if value else 0

        # Make sure our canaries report back properly.
        for name, should_exist in canaries.items():
            if should_exist != results[name]:
                print('\nWe %s `%s` in the libraries.' % (
                    'didn\'t find' if should_exist else 'found',
                    name
                ))
                print('We look for it only as a sanity check to make sure the build\n'
                      'process is working as expected. It is not, so we must abort.\n'
                      '\n'
                      'Please open a ticket at https://github.com/mikeboers/PyAV/issues\n'
                      'with the folowing information:\n')
                dump_config()
                exit(1)


class DoctorCommand(Command):

    user_options = []
    def initialize_options(self):
        pass
    def finalize_options(self):
        pass

    def run(self):
        self.run_command('config')
        self.run_command('reflect')
        print()
        dump_config()


class CleanCommand(clean):

    user_options = clean.user_options + [
        ('sources', None,
         "remove Cython build output (C sources)")]

    boolean_options = clean.boolean_options + ['sources']

    def initialize_options(self):
        clean.initialize_options(self)
        self.sources = None

    def run(self):
        clean.run(self)
        if self.sources:
            if os.path.exists('src'):
                remove_tree('src', dry_run=self.dry_run)
            else:
                log.info("'%s' does not exist -- can't clean it", 'src')


class CythonizeCommand(Command):

    user_options = []
    def initialize_options(self):
        pass
    def finalize_options(self):
        pass

    def run(self):

        # Cythonize, if required. We do it individually since we must update
        # the existing extension instead of replacing them all.
        for i, ext in enumerate(self.distribution.ext_modules):
            if any(s.endswith('.pyx') for s in ext.sources):
                if is_msvc():
                    ext.define_macros.append(('inline', '__inline'))
                new_ext = cythonize(
                    ext,
                    compiler_directives=dict(
                        c_string_type='str',
                        c_string_encoding='ascii',
                        embedsignature=True,
                    ),
                    build_dir='src',
                    include_path=ext.include_dirs,
                )[0]
                ext.sources = new_ext.sources


class BuildExtCommand(build_ext):

    if os.name != 'nt':
        user_options = build_ext.user_options + [
            ('no-pkg-config', None,
             "do not use pkg-config to configure dependencies")]

        boolean_options = build_ext.boolean_options + ['no-pkg-config']

        def initialize_options(self):
            build_ext.initialize_options(self)
            self.no_pkg_config = None
    else:
        no_pkg_config = 1

    def run(self):

        # Propagate build options to reflect
        obj = self.distribution.get_command_obj('reflect')
        obj.compiler = self.compiler
        obj.no_pkg_config = self.no_pkg_config
        obj.include_dirs = self.include_dirs
        obj.libraries = self.libraries
        obj.library_dirs = self.library_dirs

        self.run_command('reflect')

        # We write a header file containing everything we have discovered by
        # inspecting the libraries which exist. This is the main mechanism we
        # use to detect differenced between FFmpeg and Libav.

        include_dir = os.path.join(self.build_temp, 'include')
        pyav_dir = os.path.join(include_dir, 'pyav')
        try:
            os.makedirs(pyav_dir)
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise
        header_path = os.path.join(pyav_dir, 'config.h')
        print('writing', header_path)
        with open(header_path, 'w') as fh:
            fh.write('#ifndef PYAV_COMPAT_H\n')
            fh.write('#define PYAV_COMPAT_H\n')
            for k, v in sorted(config_macros.items()):
                fh.write('#define %s %s\n' % (k, v))
            fh.write('#endif\n')

        self.include_dirs = self.include_dirs or []
        self.include_dirs.append(include_dir)
        # Propagate config to cythonize.
        for i, ext in enumerate(self.distribution.ext_modules):
            unique_extend(ext.include_dirs, self.include_dirs)
            unique_extend(ext.library_dirs, self.library_dirs)
            unique_extend(ext.libraries, self.libraries)

        self.run_command('cythonize')

        return build_ext.run(self)


setup(

    name='av',
    version=version,
    description='Pythonic bindings for FFmpeg/Libav.',

    author="Mike Boers",
    author_email="pyav@mikeboers.com",

    url="https://github.com/mikeboers/PyAV",

    packages=find_packages(exclude=['build*', 'tests*', 'examples*']),

    zip_safe=False,
    ext_modules=ext_modules,

    cmdclass={
        'build_ext': BuildExtCommand,
        'clean': CleanCommand,
        'config': ConfigCommand,
        'cythonize': CythonizeCommand,
        'doctor': DoctorCommand,
        'reflect': ReflectCommand,
    },

    test_suite='nose.collector',

    entry_points={
        'console_scripts': [
            'pyav = av.__main__:main',
        ],
    },

    classifiers=[
       'Development Status :: 3 - Alpha',
       'Intended Audience :: Developers',
       'License :: OSI Approved :: BSD License',
       'Natural Language :: English',
       'Operating System :: MacOS :: MacOS X',
       'Operating System :: POSIX',
       'Operating System :: Unix',
       'Programming Language :: Cython',
       'Programming Language :: Python :: 2.6',
       'Programming Language :: Python :: 2.7',
       'Programming Language :: Python :: 3.3',
       'Programming Language :: Python :: 3.4',
       'Topic :: Software Development :: Libraries :: Python Modules',
       'Topic :: Multimedia :: Sound/Audio',
       'Topic :: Multimedia :: Sound/Audio :: Conversion',
       'Topic :: Multimedia :: Video',
       'Topic :: Multimedia :: Video :: Conversion',
   ],

    distclass=distclass,

)