def build_man_pages(self, dest, compress=False): from calibre.utils.localization import available_translations dest = os.path.abspath(dest) if os.path.exists(dest): shutil.rmtree(dest) os.makedirs(dest) base = self.j(self.d(self.SRC), 'manual') languages = list(available_translations()) languages = ['en'] + list(set(languages) - {'en', 'en_GB'}) os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages) try: os.makedirs(dest) except EnvironmentError: pass jobs = [] for l in languages: jobs.append( create_job([ sys.executable, self.j(base, 'build.py'), '--man-pages', l, dest ], '\n\n**************** Building translations for: %s' % l)) self.info('\tCreating man pages in {} for {} languages...'.format( dest, len(jobs))) subprocess.check_call(jobs[0].cmd) if not parallel_build(jobs[1:], self.info, verbose=False): raise SystemExit(1) cwd = os.getcwd() os.chdir(dest) try: for x in tuple(os.listdir('.')): if x in languages: if x == 'en': os.rename(x, 'man1') else: os.mkdir(self.j(x, 'man1')) for y in os.listdir(x): if y != 'man1': os.rename(self.j(x, y), self.j(x, 'man1', y)) else: shutil.rmtree(x) if os.path.isdir(x) else os.remove(x) if compress: jobs = [] for dirpath, dirnames, filenames in os.walk('.'): for f in filenames: if f.endswith('.1'): jobs.append( create_job( ['gzip', '--best', self.j(dirpath, f)], '')) if not parallel_build(jobs, self.info, verbose=False): raise SystemExit(1) finally: os.chdir(cwd)
def build_launchers(self): self.obj_dir = self.j(self.src_root, 'build', 'launcher') if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) base = self.j(self.src_root, 'setup', 'installer', 'linux') sources = [self.j(base, x) for x in ['util.c']] headers = [self.j(base, x) for x in ['util.h']] objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources] cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%py_ver cflags = cflags.split() + ['-I%s/include/python%s' % (SW, py_ver)] for src, obj in zip(sources, objects): if not self.newer(obj, headers+[src, __file__]): continue cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] self.run_builder(cmd) dll = self.j(self.lib_dir, 'libcalibre-launcher.so') if self.newer(dll, objects): cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \ ['-L%s/lib'%SW, '-lpython'+py_ver] self.info('Linking libcalibre-launcher.so') self.run_builder(cmd) src = self.j(base, 'main.c') modules['console'].append('calibre.linux') basenames['console'].append('calibre_postinstall') functions['console'].append('main') c_launcher = '/tmp/calibre-c-launcher' lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher.c') cmd = ['gcc', '-O2', '-DMAGICK_BASE="%s"' % self.magick_base, '-o', c_launcher, lsrc, ] self.info('Compiling launcher') self.run_builder(cmd, verbose=False) jobs = [] self.info('Processing launchers') for typ in ('console', 'gui', ): for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]): xflags = list(cflags) xflags.remove('-c') xflags += ['-DGUI_APP='+('1' if typ == 'gui' else '0')] xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname, '-DFUNCTION="%s"'%func] exe = self.j(self.bin_dir, bname) if self.newer(exe, [src, __file__]+headers): cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + self.lib_dir, '-lcalibre-launcher'] jobs.append(create_job(cmd)) sh = self.j(self.base, bname) shutil.copy2(c_launcher, sh) os.chmod(sh, stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) if jobs: if not parallel_build(jobs, self.info, verbose=False): raise SystemExit(1)
def build_launchers(self): self.obj_dir = self.j(self.src_root, 'build', 'launcher') if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) base = self.j(self.src_root, 'setup', 'installer', 'linux') sources = [self.j(base, x) for x in ['util.c']] headers = [self.j(base, x) for x in ['util.h']] objects = [self.j(self.obj_dir, self.b(x)+'.o') for x in sources] cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"'%py_ver cflags = cflags.split() + ['-I%s/include/python%s' % (SW, py_ver)] for src, obj in zip(sources, objects): if not self.newer(obj, headers+[src, __file__]): continue cmd = ['gcc'] + cflags + ['-fPIC', '-o', obj, src] self.run_builder(cmd) dll = self.j(self.lib_dir, 'libcalibre-launcher.so') if self.newer(dll, objects): cmd = ['gcc', '-O2', '-Wl,--rpath=$ORIGIN/../lib', '-fPIC', '-o', dll, '-shared'] + objects + \ ['-L%s/lib'%SW, '-lpython'+py_ver] self.info('Linking libcalibre-launcher.so') self.run_builder(cmd) src = self.j(base, 'main.c') modules['console'].append('calibre.linux') basenames['console'].append('calibre_postinstall') functions['console'].append('main') c_launcher = '/tmp/calibre-c-launcher' lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'launcher.c') cmd = ['gcc', '-O2', '-o', c_launcher, lsrc, ] self.info('Compiling launcher') self.run_builder(cmd, verbose=False) jobs = [] self.info('Processing launchers') for typ in ('console', 'gui', ): for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]): xflags = list(cflags) xflags.remove('-c') xflags += ['-DGUI_APP='+('1' if typ == 'gui' else '0')] xflags += ['-DMODULE="%s"'%mod, '-DBASENAME="%s"'%bname, '-DFUNCTION="%s"'%func] exe = self.j(self.bin_dir, bname) if self.newer(exe, [src, __file__]+headers): cmd = ['gcc'] + xflags + [src, '-o', exe, '-L' + self.lib_dir, '-lcalibre-launcher'] jobs.append(create_job(cmd)) sh = self.j(self.base, bname) shutil.copy2(c_launcher, sh) os.chmod(sh, stat.S_IREAD|stat.S_IEXEC|stat.S_IWRITE|stat.S_IRGRP|stat.S_IXGRP|stat.S_IROTH|stat.S_IXOTH) if jobs: if not parallel_build(jobs, self.info, verbose=False): raise SystemExit(1)
def build(self, ext, dest): from setup.parallel_build import create_job, parallel_build if ext.sip_files: return self.build_pyqt_extension(ext, dest) compiler = self.env.cxx if ext.needs_cxx else self.env.cc linker = self.env.linker if iswindows else compiler objects = [] obj_dir = self.j(self.obj_dir, ext.name) einc = self.inc_dirs_to_cflags(ext.inc_dirs) if not os.path.exists(obj_dir): os.makedirs(obj_dir) jobs = [] for src in ext.sources: obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + '.o') objects.append(obj) if self.newer(obj, [src] + ext.headers): inf = '/Tp' if src.endswith('.cpp') or src.endswith( '.cxx') else '/Tc' sinc = [inf + src] if iswindows else ['-c', src] oinc = ['/Fo' + obj] if iswindows else ['-o', obj] cmd = [compiler ] + self.env.cflags + ext.cflags + einc + sinc + oinc jobs.append(create_job(cmd)) if jobs: self.info('Compiling', ext.name) if not parallel_build(jobs, self.info): raise SystemExit(1) dest = self.dest(ext) elib = self.lib_dirs_to_ldflags(ext.lib_dirs) xlib = self.libraries_to_ldflags(ext.libraries) if self.newer(dest, objects + ext.extra_objs): self.info('Linking', ext.name) cmd = [linker] if iswindows: pre_ld_flags = [] if ext.name in ('icu', 'matcher'): # windows has its own ICU libs that dont work pre_ld_flags = elib cmd += pre_ld_flags + self.env.ldflags + ext.ldflags + elib + xlib + \ ['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest] else: cmd += objects + ext.extra_objs + [ '-o', dest ] + self.env.ldflags + ext.ldflags + elib + xlib self.info('\n\n', ' '.join(cmd), '\n\n') self.check_call(cmd) if iswindows: for x in ('.exp', '.lib'): x = os.path.splitext(dest)[0] + x if os.path.exists(x): os.remove(x)
def build(self, ext, dest): if ext.sip_files: return self.build_pyqt_extension(ext, dest) compiler = cxx if ext.needs_cxx else cc linker = msvc.linker if iswindows else compiler objects = [] obj_dir = self.j(self.obj_dir, ext.name) ext.preflight(obj_dir, compiler, linker, self, cflags, ldflags) einc = self.inc_dirs_to_cflags(ext.inc_dirs) if not os.path.exists(obj_dir): os.makedirs(obj_dir) jobs = [] for src in ext.sources: obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + ".o") objects.append(obj) if self.newer(obj, [src] + ext.headers): inf = "/Tp" if src.endswith(".cpp") or src.endswith(".cxx") else "/Tc" sinc = [inf + src] if iswindows else ["-c", src] oinc = ["/Fo" + obj] if iswindows else ["-o", obj] cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc jobs.append(create_job(cmd)) if jobs: self.info("Compiling", ext.name) if not parallel_build(jobs, self.info): raise SystemExit(1) dest = self.dest(ext) elib = self.lib_dirs_to_ldflags(ext.lib_dirs) xlib = self.libraries_to_ldflags(ext.libraries) if self.newer(dest, objects + ext.extra_objs): self.info("Linking", ext.name) cmd = [linker] if iswindows: cmd += ( ldflags + ext.ldflags + elib + xlib + ["/EXPORT:init" + ext.name] + objects + ext.extra_objs + ["/OUT:" + dest] ) else: cmd += objects + ext.extra_objs + ["-o", dest] + ldflags + ext.ldflags + elib + xlib self.info("\n\n", " ".join(cmd), "\n\n") self.check_call(cmd) if iswindows: for x in (".exp", ".lib"): x = os.path.splitext(dest)[0] + x if os.path.exists(x): os.remove(x)
def run(self, opts): tdir = self.j(tempfile.gettempdir(), 'user-manual-build') if os.path.exists(tdir): shutil.rmtree(tdir) os.mkdir(tdir) st = time.time() base = self.j(self.d(self.SRC), 'manual') for d in ('generated', ): d = self.j(base, d) if os.path.exists(d): shutil.rmtree(d) os.makedirs(d) jobs = [] languages = opts.language or list( json.load(open(self.j(base, 'locale', 'completed.json'), 'rb'))) languages = ['en'] + list(set(languages) - {'en'}) os.environ['ALL_USER_MANUAL_LANGUAGES'] = ' '.join(languages) for language in languages: jobs.append( create_job([ sys.executable, self.j(self.d(self.SRC), 'manual', 'build.py'), language, self.j(tdir, language) ], '\n\n**************** Building translations for: %s' % language)) self.info('Building manual for %d languages' % len(jobs)) subprocess.check_call(jobs[0].cmd) if not parallel_build(jobs[1:], self.info): raise SystemExit(1) cwd = os.getcwd() with open('resources/localization/website-languages.txt') as wl: languages = frozenset( filter(None, (x.strip() for x in wl.read().split()))) try: os.chdir(self.j(tdir, 'en', 'html')) for x in os.listdir(tdir): if x != 'en': shutil.copytree(self.j(tdir, x, 'html'), x) self.replace_with_symlinks(x) else: os.symlink('.', 'en') for x in languages: if x and not os.path.exists(x): os.symlink('.', x) self.info('Built manual for %d languages in %s minutes' % (len(jobs), int((time.time() - st) / 60.))) finally: os.chdir(cwd) if opts.serve: self.serve_manual(self.j(tdir, 'en', 'html'))
def build(self, ext, dest): if ext.sip_files: return self.build_pyqt_extension(ext, dest) compiler = cxx if ext.needs_cxx else cc linker = msvc.linker if iswindows else compiler objects = [] obj_dir = self.j(self.obj_dir, ext.name) ext.preflight(obj_dir, compiler, linker, self, cflags, ldflags) einc = self.inc_dirs_to_cflags(ext.inc_dirs) if not os.path.exists(obj_dir): os.makedirs(obj_dir) jobs = [] for src in ext.sources: obj = self.j(obj_dir, os.path.splitext(self.b(src))[0] + '.o') objects.append(obj) if self.newer(obj, [src] + ext.headers): inf = '/Tp' if src.endswith('.cpp') or src.endswith( '.cxx') else '/Tc' sinc = [inf + src] if iswindows else ['-c', src] oinc = ['/Fo' + obj] if iswindows else ['-o', obj] cmd = [compiler] + cflags + ext.cflags + einc + sinc + oinc jobs.append(create_job(cmd)) if jobs: self.info('Compiling', ext.name) if not parallel_build(jobs, self.info): raise SystemExit(1) dest = self.dest(ext) elib = self.lib_dirs_to_ldflags(ext.lib_dirs) xlib = self.libraries_to_ldflags(ext.libraries) if self.newer(dest, objects + ext.extra_objs): self.info('Linking', ext.name) cmd = [linker] if iswindows: cmd += ldflags + ext.ldflags + elib + xlib + \ ['/EXPORT:init'+ext.name] + objects + ext.extra_objs + ['/OUT:'+dest] else: cmd += objects + ext.extra_objs + [ '-o', dest ] + ldflags + ext.ldflags + elib + xlib self.info('\n\n', ' '.join(cmd), '\n\n') self.check_call(cmd) if iswindows: for x in ('.exp', '.lib'): x = os.path.splitext(dest)[0] + x if os.path.exists(x): os.remove(x)
def build(self, ext, dest): from setup.parallel_build import create_job, parallel_build if ext.sip_files: return self.build_pyqt_extension(ext, dest) compiler = self.env.cxx if ext.needs_cxx else self.env.cc linker = self.env.linker if iswindows else compiler objects = [] obj_dir = self.j(self.obj_dir, ext.name) einc = self.inc_dirs_to_cflags(ext.inc_dirs) if not os.path.exists(obj_dir): os.makedirs(obj_dir) jobs = [] for src in ext.sources: obj = self.j(obj_dir, os.path.splitext(self.b(src))[0]+'.o') objects.append(obj) if self.newer(obj, [src]+ext.headers): inf = '/Tp' if src.endswith('.cpp') or src.endswith('.cxx') else '/Tc' sinc = [inf+src] if iswindows else ['-c', src] oinc = ['/Fo'+obj] if iswindows else ['-o', obj] cmd = [compiler] + self.env.cflags + ext.cflags + einc + sinc + oinc jobs.append(create_job(cmd)) if jobs: self.info('Compiling', ext.name) if not parallel_build(jobs, self.info): raise SystemExit(1) dest = self.dest(ext) elib = self.lib_dirs_to_ldflags(ext.lib_dirs) xlib = self.libraries_to_ldflags(ext.libraries) if self.newer(dest, objects+ext.extra_objs): self.info('Linking', ext.name) cmd = [linker] if iswindows: cmd += self.env.ldflags + ext.ldflags + elib + xlib + \ ['/EXPORT:' + init_symbol_name(ext.name)] + objects + ext.extra_objs + ['/OUT:'+dest] else: cmd += objects + ext.extra_objs + ['-o', dest] + self.env.ldflags + ext.ldflags + elib + xlib self.info('\n\n', ' '.join(cmd), '\n\n') self.check_call(cmd) if iswindows: for x in ('.exp', '.lib'): x = os.path.splitext(dest)[0]+x if os.path.exists(x): os.remove(x)
def run(self, opts): from setup.parallel_build import parallel_build, create_job if opts.no_compile: self.info('--no-compile specified, skipping compilation') return self.env = init_env(debug=opts.debug) all_extensions = map(parse_extension, filter(is_ext_allowed, read_extensions())) self.build_dir = os.path.abspath(opts.build_dir or self.DEFAULT_BUILDDIR) self.output_dir = os.path.abspath(opts.output_dir or self.DEFAULT_OUTPUTDIR) self.obj_dir = os.path.join(self.build_dir, 'objects') for x in (self.output_dir, self.obj_dir): os.makedirs(x, exist_ok=True) pyqt_extensions, extensions = [], [] for ext in all_extensions: if opts.only != 'all' and opts.only != ext.name: continue if ext.error: if ext.optional: self.warn(ext.error) continue else: raise Exception(ext.error) dest = self.dest(ext) os.makedirs(self.d(dest), exist_ok=True) (pyqt_extensions if ext.sip_files else extensions).append((ext, dest)) jobs = [] objects_map = {} self.info(f'Building {len(extensions)+len(pyqt_extensions)} extensions') ccdb = [] for (ext, dest) in extensions: cmds, objects = self.get_compile_commands(ext, dest, ccdb) objects_map[id(ext)] = objects for cmd in cmds: jobs.append(create_job(cmd.cmd)) self.dump_db('compile', ccdb) if jobs: self.info(f'Compiling {len(jobs)} files...') if not parallel_build(jobs, self.info): raise SystemExit(1) jobs, link_commands, lddb = [], [], [] for (ext, dest) in extensions: objects = objects_map[id(ext)] cmd = self.get_link_command(ext, dest, objects, lddb) if cmd is not None: link_commands.append(cmd) jobs.append(create_job(cmd.cmd)) self.dump_db('link', lddb) if jobs: self.info(f'Linking {len(jobs)} files...') if not parallel_build(jobs, self.info): raise SystemExit(1) for cmd in link_commands: self.post_link_cleanup(cmd) jobs = [] sbf_map = {} for (ext, dest) in pyqt_extensions: cmd, sbf, cwd = self.get_sip_commands(ext) sbf_map[id(ext)] = sbf if cmd is not None: jobs.append(create_job(cmd, cwd=cwd)) if jobs: self.info(f'SIPing {len(jobs)} files...') if not parallel_build(jobs, self.info): raise SystemExit(1) for (ext, dest) in pyqt_extensions: sbf = sbf_map[id(ext)] if not os.path.exists(sbf): self.build_pyqt_extension(ext, dest, sbf) if opts.only in {'all', 'headless'}: self.build_headless()
def build_launchers(self): self.obj_dir = self.j(self.src_root, "build", "launcher") if not os.path.exists(self.obj_dir): os.makedirs(self.obj_dir) base = self.j(self.src_root, "setup", "installer", "linux") sources = [self.j(base, x) for x in ["util.c"]] headers = [self.j(base, x) for x in ["util.h"]] objects = [self.j(self.obj_dir, self.b(x) + ".o") for x in sources] cflags = '-fno-strict-aliasing -W -Wall -c -O2 -pipe -DPYTHON_VER="python%s"' % py_ver cflags = cflags.split() + ["-I%s/include/python%s" % (SW, py_ver)] for src, obj in zip(sources, objects): if not self.newer(obj, headers + [src, __file__]): continue cmd = ["gcc"] + cflags + ["-fPIC", "-o", obj, src] self.run_builder(cmd) dll = self.j(self.lib_dir, "libcalibre-launcher.so") if self.newer(dll, objects): cmd = ( ["gcc", "-O2", "-Wl,--rpath=$ORIGIN/../lib", "-fPIC", "-o", dll, "-shared"] + objects + ["-L%s/lib" % SW, "-lpython" + py_ver] ) self.info("Linking libcalibre-launcher.so") self.run_builder(cmd) src = self.j(base, "main.c") modules["console"].append("calibre.linux") basenames["console"].append("calibre_postinstall") functions["console"].append("main") c_launcher = "/tmp/calibre-c-launcher" lsrc = os.path.join(os.path.dirname(os.path.abspath(__file__)), "launcher.c") cmd = ["gcc", "-O2", '-DMAGICK_BASE="%s"' % self.magick_base, "-o", c_launcher, lsrc] self.info("Compiling launcher") self.run_builder(cmd, verbose=False) jobs = [] self.info("Processing launchers") for typ in ("console", "gui"): for mod, bname, func in zip(modules[typ], basenames[typ], functions[typ]): xflags = list(cflags) xflags.remove("-c") xflags += ["-DGUI_APP=" + ("1" if typ == "gui" else "0")] xflags += ['-DMODULE="%s"' % mod, '-DBASENAME="%s"' % bname, '-DFUNCTION="%s"' % func] exe = self.j(self.bin_dir, bname) if self.newer(exe, [src, __file__] + headers): cmd = ["gcc"] + xflags + [src, "-o", exe, "-L" + self.lib_dir, "-lcalibre-launcher"] jobs.append(create_job(cmd)) sh = self.j(self.base, bname) shutil.copy2(c_launcher, sh) os.chmod( sh, stat.S_IREAD | stat.S_IEXEC | stat.S_IWRITE | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH, ) if jobs: if not parallel_build(jobs, self.info, verbose=False): raise SystemExit(1)