Exemplo n.º 1
0
 def build_man_pages(self, dest, compress=False):
     from calibre.utils.localization import available_translations
     dest = os.path.abspath(dest)
     if os.path.exists(dest):
         shutil.rmtree(dest)
     os.makedirs(dest)
     base = self.j(self.d(self.SRC), 'manual')
     languages = list(available_translations())
     languages = ['en'] + list(set(languages) - {'en', 'en_GB'})
     os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages)
     try:
         os.makedirs(dest)
     except EnvironmentError:
         pass
     jobs = []
     for l in languages:
         jobs.append(
             create_job([
                 sys.executable,
                 self.j(base, 'build.py'), '--man-pages', l, dest
             ], '\n\n**************** Building translations for: %s' % l))
     self.info('\tCreating man pages in {} for {} languages...'.format(
         dest, len(jobs)))
     subprocess.check_call(jobs[0].cmd)
     if not parallel_build(jobs[1:], self.info, verbose=False):
         raise SystemExit(1)
     cwd = os.getcwd()
     os.chdir(dest)
     try:
         for x in tuple(os.listdir('.')):
             if x in languages:
                 if x == 'en':
                     os.rename(x, 'man1')
                 else:
                     os.mkdir(self.j(x, 'man1'))
                     for y in os.listdir(x):
                         if y != 'man1':
                             os.rename(self.j(x, y), self.j(x, 'man1', y))
             else:
                 shutil.rmtree(x) if os.path.isdir(x) else os.remove(x)
         if compress:
             jobs = []
             for dirpath, dirnames, filenames in os.walk('.'):
                 for f in filenames:
                     if f.endswith('.1'):
                         jobs.append(
                             create_job(
                                 ['gzip', '--best',
                                  self.j(dirpath, f)], ''))
             if not parallel_build(jobs, self.info, verbose=False):
                 raise SystemExit(1)
     finally:
         os.chdir(cwd)
Exemplo n.º 2
0
    def build_launchers(self):
        self.obj_dir = self.j(self.src_root, 'build', 'launcher')
        if not os.path.exists(self.obj_dir):
            os.makedirs(self.obj_dir)
        base = self.j(self.src_root, 'setup', 'installer', 'linux')
        sources = [self.j(base, x) for x in ['util.c']]
        headers = [self.j(base, x) for x in ['util.h']]
        objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources]
        cflags  = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%py_ver
        cflags  = cflags.split() + ['-I%s/include/python%s' % (SW, py_ver)]
        for src, obj in zip(sources, objects):
            if not self.newer(obj, headers+[src, __file__]):
                continue
            cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
            self.run_builder(cmd)

        dll = self.j(self.lib_dir, 'libcalibre-launcher.so')
        if self.newer(dll, objects):
            cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \
                    ['-L%s/lib'%SW, '-lpython'+py_ver]
            self.info('Linking libcalibre-launcher.so')
            self.run_builder(cmd)

        src = self.j(base, 'main.c')

        modules['console'].append('calibre.linux')
        basenames['console'].append('calibre_postinstall')
        functions['console'].append('main')
        c_launcher = '/tmp/calibre-c-launcher'
        lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher.c')
        cmd = ['gcc', '-O2', '-DMAGICK_BASE="%s"' % self.magick_base, '-o', c_launcher, lsrc, ]
        self.info('Compiling launcher')
        self.run_builder(cmd, verbose=False)

        jobs = []
        self.info('Processing launchers')
        for typ in ('console', 'gui', ):
            for mod, bname, func in zip(modules[typ], basenames[typ],
                    functions[typ]):
                xflags = list(cflags)
                xflags.remove('-c')
                xflags += ['-DGUI_APP='+('1' if typ == 'gui' else '0')]
                xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname,
                    '-DFUNCTION="%s"'%func]

                exe = self.j(self.bin_dir, bname)
                if self.newer(exe, [src, __file__]+headers):
                    cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + self.lib_dir, '-lcalibre-launcher']
                    jobs.append(create_job(cmd))
                sh = self.j(self.base, bname)
                shutil.copy2(c_launcher, sh)
                os.chmod(sh,
                    stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
        if jobs:
            if not parallel_build(jobs, self.info, verbose=False):
                raise SystemExit(1)
Exemplo n.º 3
0
    def build_launchers(self):
        self.obj_dir = self.j(self.src_root, 'build', 'launcher')
        if not os.path.exists(self.obj_dir):
            os.makedirs(self.obj_dir)
        base = self.j(self.src_root, 'setup', 'installer', 'linux')
        sources = [self.j(base, x) for x in ['util.c']]
        headers = [self.j(base, x) for x in ['util.h']]
        objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources]
        cflags  = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%py_ver
        cflags  = cflags.split() + ['-I%s/include/python%s' % (SW, py_ver)]
        for src, obj in zip(sources, objects):
            if not self.newer(obj, headers+[src, __file__]):
                continue
            cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src]
            self.run_builder(cmd)

        dll = self.j(self.lib_dir, 'libcalibre-launcher.so')
        if self.newer(dll, objects):
            cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \
                    ['-L%s/lib'%SW, '-lpython'+py_ver]
            self.info('Linking libcalibre-launcher.so')
            self.run_builder(cmd)

        src = self.j(base, 'main.c')

        modules['console'].append('calibre.linux')
        basenames['console'].append('calibre_postinstall')
        functions['console'].append('main')
        c_launcher = '/tmp/calibre-c-launcher'
        lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher.c')
        cmd = ['gcc', '-O2', '-o', c_launcher, lsrc, ]
        self.info('Compiling launcher')
        self.run_builder(cmd, verbose=False)

        jobs = []
        self.info('Processing launchers')
        for typ in ('console', 'gui', ):
            for mod, bname, func in zip(modules[typ], basenames[typ],
                    functions[typ]):
                xflags = list(cflags)
                xflags.remove('-c')
                xflags += ['-DGUI_APP='+('1' if typ == 'gui' else '0')]
                xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname,
                    '-DFUNCTION="%s"'%func]

                exe = self.j(self.bin_dir, bname)
                if self.newer(exe, [src, __file__]+headers):
                    cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + self.lib_dir, '-lcalibre-launcher']
                    jobs.append(create_job(cmd))
                sh = self.j(self.base, bname)
                shutil.copy2(c_launcher, sh)
                os.chmod(sh,
                    stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH)
        if jobs:
            if not parallel_build(jobs, self.info, verbose=False):
                raise SystemExit(1)
Exemplo n.º 4
0
    def build(self, ext, dest):
        from setup.parallel_build import create_job, parallel_build
        if ext.sip_files:
            return self.build_pyqt_extension(ext, dest)
        compiler = self.env.cxx if ext.needs_cxx else self.env.cc
        linker = self.env.linker if iswindows else compiler
        objects = []
        obj_dir = self.j(self.obj_dir, ext.name)
        einc = self.inc_dirs_to_cflags(ext.inc_dirs)
        if not os.path.exists(obj_dir):
            os.makedirs(obj_dir)

        jobs = []
        for src in ext.sources:
            obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + '.o')
            objects.append(obj)
            if self.newer(obj, [src] + ext.headers):
                inf = '/Tp' if src.endswith('.cpp') or src.endswith(
                    '.cxx') else '/Tc'
                sinc = [inf + src] if iswindows else ['-c', src]
                oinc = ['/Fo' + obj] if iswindows else ['-o', obj]
                cmd = [compiler
                       ] + self.env.cflags + ext.cflags + einc + sinc + oinc
                jobs.append(create_job(cmd))
        if jobs:
            self.info('Compiling', ext.name)
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)

        dest = self.dest(ext)
        elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
        xlib = self.libraries_to_ldflags(ext.libraries)
        if self.newer(dest, objects + ext.extra_objs):
            self.info('Linking', ext.name)
            cmd = [linker]
            if iswindows:
                pre_ld_flags = []
                if ext.name in ('icu', 'matcher'):
                    # windows has its own ICU libs that dont work
                    pre_ld_flags = elib
                cmd += pre_ld_flags + self.env.ldflags + ext.ldflags + elib + xlib + \
                    ['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest]
            else:
                cmd += objects + ext.extra_objs + [
                    '-o', dest
                ] + self.env.ldflags + ext.ldflags + elib + xlib
            self.info('\n\n', ' '.join(cmd), '\n\n')
            self.check_call(cmd)
            if iswindows:
                for x in ('.exp', '.lib'):
                    x = os.path.splitext(dest)[0] + x
                    if os.path.exists(x):
                        os.remove(x)
Exemplo n.º 5
0
    def build(self, ext, dest):
        if ext.sip_files:
            return self.build_pyqt_extension(ext, dest)
        compiler = cxx if ext.needs_cxx else cc
        linker = msvc.linker if iswindows else compiler
        objects = []
        obj_dir = self.j(self.obj_dir, ext.name)
        ext.preflight(obj_dir, compiler, linker, self, cflags, ldflags)
        einc = self.inc_dirs_to_cflags(ext.inc_dirs)
        if not os.path.exists(obj_dir):
            os.makedirs(obj_dir)

        jobs = []
        for src in ext.sources:
            obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + ".o")
            objects.append(obj)
            if self.newer(obj, [src] + ext.headers):
                inf = "/Tp" if src.endswith(".cpp") or src.endswith(".cxx") else "/Tc"
                sinc = [inf + src] if iswindows else ["-c", src]
                oinc = ["/Fo" + obj] if iswindows else ["-o", obj]
                cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc
                jobs.append(create_job(cmd))
        if jobs:
            self.info("Compiling", ext.name)
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)

        dest = self.dest(ext)
        elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
        xlib = self.libraries_to_ldflags(ext.libraries)
        if self.newer(dest, objects + ext.extra_objs):
            self.info("Linking", ext.name)
            cmd = [linker]
            if iswindows:
                cmd += (
                    ldflags
                    + ext.ldflags
                    + elib
                    + xlib
                    + ["/EXPORT:init" + ext.name]
                    + objects
                    + ext.extra_objs
                    + ["/OUT:" + dest]
                )
            else:
                cmd += objects + ext.extra_objs + ["-o", dest] + ldflags + ext.ldflags + elib + xlib
            self.info("\n\n", " ".join(cmd), "\n\n")
            self.check_call(cmd)
            if iswindows:
                for x in (".exp", ".lib"):
                    x = os.path.splitext(dest)[0] + x
                    if os.path.exists(x):
                        os.remove(x)
Exemplo n.º 6
0
    def run(self, opts):
        tdir = self.j(tempfile.gettempdir(), 'user-manual-build')
        if os.path.exists(tdir):
            shutil.rmtree(tdir)
        os.mkdir(tdir)
        st = time.time()
        base = self.j(self.d(self.SRC), 'manual')
        for d in ('generated', ):
            d = self.j(base, d)
            if os.path.exists(d):
                shutil.rmtree(d)
            os.makedirs(d)
        jobs = []
        languages = opts.language or list(
            json.load(open(self.j(base, 'locale', 'completed.json'), 'rb')))
        languages = ['en'] + list(set(languages) - {'en'})
        os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages)
        for language in languages:
            jobs.append(
                create_job([
                    sys.executable,
                    self.j(self.d(self.SRC), 'manual', 'build.py'), language,
                    self.j(tdir, language)
                ], '\n\n**************** Building translations for: %s' %
                           language))
        self.info('Building manual for %d languages' % len(jobs))
        subprocess.check_call(jobs[0].cmd)
        if not parallel_build(jobs[1:], self.info):
            raise SystemExit(1)
        cwd = os.getcwd()
        with open('resources/localization/website-languages.txt') as wl:
            languages = frozenset(
                filter(None, (x.strip() for x in wl.read().split())))
        try:
            os.chdir(self.j(tdir, 'en', 'html'))
            for x in os.listdir(tdir):
                if x != 'en':
                    shutil.copytree(self.j(tdir, x, 'html'), x)
                    self.replace_with_symlinks(x)
                else:
                    os.symlink('.', 'en')
            for x in languages:
                if x and not os.path.exists(x):
                    os.symlink('.', x)
            self.info('Built manual for %d languages in %s minutes' %
                      (len(jobs), int((time.time() - st) / 60.)))
        finally:
            os.chdir(cwd)

        if opts.serve:
            self.serve_manual(self.j(tdir, 'en', 'html'))
Exemplo n.º 7
0
    def build(self, ext, dest):
        if ext.sip_files:
            return self.build_pyqt_extension(ext, dest)
        compiler = cxx if ext.needs_cxx else cc
        linker = msvc.linker if iswindows else compiler
        objects = []
        obj_dir = self.j(self.obj_dir, ext.name)
        ext.preflight(obj_dir, compiler, linker, self, cflags, ldflags)
        einc = self.inc_dirs_to_cflags(ext.inc_dirs)
        if not os.path.exists(obj_dir):
            os.makedirs(obj_dir)

        jobs = []
        for src in ext.sources:
            obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + '.o')
            objects.append(obj)
            if self.newer(obj, [src] + ext.headers):
                inf = '/Tp' if src.endswith('.cpp') or src.endswith(
                    '.cxx') else '/Tc'
                sinc = [inf + src] if iswindows else ['-c', src]
                oinc = ['/Fo' + obj] if iswindows else ['-o', obj]
                cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc
                jobs.append(create_job(cmd))
        if jobs:
            self.info('Compiling', ext.name)
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)

        dest = self.dest(ext)
        elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
        xlib = self.libraries_to_ldflags(ext.libraries)
        if self.newer(dest, objects + ext.extra_objs):
            self.info('Linking', ext.name)
            cmd = [linker]
            if iswindows:
                cmd += ldflags + ext.ldflags + elib + xlib + \
                    ['/EXPORT:init'+ext.name] + objects + ext.extra_objs + ['/OUT:'+dest]
            else:
                cmd += objects + ext.extra_objs + [
                    '-o', dest
                ] + ldflags + ext.ldflags + elib + xlib
            self.info('\n\n', ' '.join(cmd), '\n\n')
            self.check_call(cmd)
            if iswindows:
                for x in ('.exp', '.lib'):
                    x = os.path.splitext(dest)[0] + x
                    if os.path.exists(x):
                        os.remove(x)
Exemplo n.º 8
0
    def build(self, ext, dest):
        from setup.parallel_build import create_job, parallel_build
        if ext.sip_files:
            return self.build_pyqt_extension(ext, dest)
        compiler = self.env.cxx if ext.needs_cxx else self.env.cc
        linker = self.env.linker if iswindows else compiler
        objects = []
        obj_dir = self.j(self.obj_dir, ext.name)
        einc = self.inc_dirs_to_cflags(ext.inc_dirs)
        if not os.path.exists(obj_dir):
            os.makedirs(obj_dir)

        jobs = []
        for src in ext.sources:
            obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o')
            objects.append(obj)
            if self.newer(obj, [src]+ext.headers):
                inf = '/Tp' if src.endswith('.cpp') or src.endswith('.cxx') else '/Tc'
                sinc = [inf+src] if iswindows else ['-c', src]
                oinc = ['/Fo'+obj] if iswindows else ['-o', obj]
                cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc
                jobs.append(create_job(cmd))
        if jobs:
            self.info('Compiling', ext.name)
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)

        dest = self.dest(ext)
        elib = self.lib_dirs_to_ldflags(ext.lib_dirs)
        xlib = self.libraries_to_ldflags(ext.libraries)
        if self.newer(dest, objects+ext.extra_objs):
            self.info('Linking', ext.name)
            cmd = [linker]
            if iswindows:
                cmd += self.env.ldflags + ext.ldflags + elib + xlib + \
                    ['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest]
            else:
                cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib
            self.info('\n\n', ' '.join(cmd), '\n\n')
            self.check_call(cmd)
            if iswindows:
                for x in ('.exp', '.lib'):
                    x = os.path.splitext(dest)[0]+x
                    if os.path.exists(x):
                        os.remove(x)
Exemplo n.º 9
0
    def run(self, opts):
        from setup.parallel_build import parallel_build, create_job
        if opts.no_compile:
            self.info('--no-compile specified, skipping compilation')
            return
        self.env = init_env(debug=opts.debug)
        all_extensions = map(parse_extension, filter(is_ext_allowed, read_extensions()))
        self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR)
        self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR)
        self.obj_dir = os.path.join(self.build_dir, 'objects')
        for x in (self.output_dir, self.obj_dir):
            os.makedirs(x, exist_ok=True)
        pyqt_extensions, extensions = [], []
        for ext in all_extensions:
            if opts.only != 'all' and opts.only != ext.name:
                continue
            if ext.error:
                if ext.optional:
                    self.warn(ext.error)
                    continue
                else:
                    raise Exception(ext.error)
            dest = self.dest(ext)
            os.makedirs(self.d(dest), exist_ok=True)
            (pyqt_extensions if ext.sip_files else extensions).append((ext, dest))

        jobs = []
        objects_map = {}
        self.info(f'Building {len(extensions)+len(pyqt_extensions)} extensions')
        ccdb = []
        for (ext, dest) in extensions:
            cmds, objects = self.get_compile_commands(ext, dest, ccdb)
            objects_map[id(ext)] = objects
            for cmd in cmds:
                jobs.append(create_job(cmd.cmd))
        self.dump_db('compile', ccdb)
        if jobs:
            self.info(f'Compiling {len(jobs)} files...')
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)
        jobs, link_commands, lddb = [], [], []
        for (ext, dest) in extensions:
            objects = objects_map[id(ext)]
            cmd = self.get_link_command(ext, dest, objects, lddb)
            if cmd is not None:
                link_commands.append(cmd)
                jobs.append(create_job(cmd.cmd))
        self.dump_db('link', lddb)
        if jobs:
            self.info(f'Linking {len(jobs)} files...')
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)
            for cmd in link_commands:
                self.post_link_cleanup(cmd)

        jobs = []
        sbf_map = {}
        for (ext, dest) in pyqt_extensions:
            cmd, sbf, cwd = self.get_sip_commands(ext)
            sbf_map[id(ext)] = sbf
            if cmd is not None:
                jobs.append(create_job(cmd, cwd=cwd))
        if jobs:
            self.info(f'SIPing {len(jobs)} files...')
            if not parallel_build(jobs, self.info):
                raise SystemExit(1)
        for (ext, dest) in pyqt_extensions:
            sbf = sbf_map[id(ext)]
            if not os.path.exists(sbf):
                self.build_pyqt_extension(ext, dest, sbf)

        if opts.only in {'all', 'headless'}:
            self.build_headless()
Exemplo n.º 10
0
    def build_launchers(self):
        self.obj_dir = self.j(self.src_root, "build", "launcher")
        if not os.path.exists(self.obj_dir):
            os.makedirs(self.obj_dir)
        base = self.j(self.src_root, "setup", "installer", "linux")
        sources = [self.j(base, x) for x in ["util.c"]]
        headers = [self.j(base, x) for x in ["util.h"]]
        objects = [self.j(self.obj_dir, self.b(x) + ".o") for x in sources]
        cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"' % py_ver
        cflags = cflags.split() + ["-I%s/include/python%s" % (SW, py_ver)]
        for src, obj in zip(sources, objects):
            if not self.newer(obj, headers + [src, __file__]):
                continue
            cmd = ["gcc"] + cflags + ["-fPIC", "-o", obj, src]
            self.run_builder(cmd)

        dll = self.j(self.lib_dir, "libcalibre-launcher.so")
        if self.newer(dll, objects):
            cmd = (
                ["gcc", "-O2", "-Wl,--rpath=$ORIGIN/../lib", "-fPIC", "-o", dll, "-shared"]
                + objects
                + ["-L%s/lib" % SW, "-lpython" + py_ver]
            )
            self.info("Linking libcalibre-launcher.so")
            self.run_builder(cmd)

        src = self.j(base, "main.c")

        modules["console"].append("calibre.linux")
        basenames["console"].append("calibre_postinstall")
        functions["console"].append("main")
        c_launcher = "/tmp/calibre-c-launcher"
        lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), "launcher.c")
        cmd = ["gcc", "-O2", '-DMAGICK_BASE="%s"' % self.magick_base, "-o", c_launcher, lsrc]
        self.info("Compiling launcher")
        self.run_builder(cmd, verbose=False)

        jobs = []
        self.info("Processing launchers")
        for typ in ("console", "gui"):
            for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]):
                xflags = list(cflags)
                xflags.remove("-c")
                xflags += ["-DGUI_APP=" + ("1" if typ == "gui" else "0")]
                xflags += ['-DMODULE="%s"' % mod, '-DBASENAME="%s"' % bname, '-DFUNCTION="%s"' % func]

                exe = self.j(self.bin_dir, bname)
                if self.newer(exe, [src, __file__] + headers):
                    cmd = ["gcc"] + xflags + [src, "-o", exe, "-L" + self.lib_dir, "-lcalibre-launcher"]
                    jobs.append(create_job(cmd))
                sh = self.j(self.base, bname)
                shutil.copy2(c_launcher, sh)
                os.chmod(
                    sh,
                    stat.S_IREAD
                    | stat.S_IEXEC
                    | stat.S_IWRITE
                    | stat.S_IRGRP
                    | stat.S_IXGRP
                    | stat.S_IROTH
                    | stat.S_IXOTH,
                )
        if jobs:
            if not parallel_build(jobs, self.info, verbose=False):
                raise SystemExit(1)