def _patch(self, context):
source_path = pj(self.buildEnv.source_dir, self.source_dir)
context.try_skip(source_path)
context.force_native_build = True
for p in self.patches:
with open(pj(SCRIPT_DIR, 'patches', p), 'r') as patch_input:
self.buildEnv.run_command("patch -p1",
source_path,
context,
input=patch_input.read())
def _configure(self, context):
# Copy kiwix-android in build dir.
kiwix_android_dep = self.buildEnv.targetsDict['kiwix-android']
if not os.path.exists(self.build_path):
shutil.copytree(kiwix_android_dep.source_path, self.build_path)
# Copy kiwix-lib application in build dir
try:
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu'),
exist_ok=True)
shutil.copy2(
pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu',
'icudt58l.dat'))
# Generate custom directory
try:
shutil.rmtree(pj(self.build_path, 'custom'))
except FileNotFoundError:
pass
os.makedirs(pj(self.build_path, 'custom'))
command = "./gen-custom-android-directory.py {custom_name} --output-dir {custom_dir}"
command = command.format(custom_name=self.target.custom_name,
custom_dir=pj(self.build_path, 'custom',
self.target.custom_name))
self.buildEnv.run_command(command, self.source_path, context)
def make_option(self):
if self.buildEnv.platform_info.build == 'win32':
return "--makefile win32/Makefile.gcc PREFIX={host}- SHARED_MODE={static} INCLUDE_PATH={include_path} LIBRARY_PATH={library_path} BINARY_PATH={binary_path}".format(
host='i686-w64-mingw32',
static="0" if self.buildEnv.platform_info.static else "1",
include_path=pj(self.buildEnv.install_dir, 'include'),
library_path=pj(self.buildEnv.install_dir,
self.buildEnv.libprefix),
binary_path=pj(self.buildEnv.install_dir, 'bin'),
)
return ""
def _get_zim_size(self):
try:
zim_size = self.buildEnv.options.zim_file_size
except AttributeError:
with open(
pj(self.source_path, self.target.custom_name,
'info.json')) as f:
app_info = json.load(f)
zim_size = os.path.getsize(
pj(self.custom_build_path, app_info['zim_file']))
return zim_size
def _gen_crossfile(self, name):
crossfile = pj(self.build_dir, name)
template_file = pj(SCRIPT_DIR, 'templates', name)
with open(template_file, 'r') as f:
template = f.read()
content = template.format(
toolchain=self.toolchains[0],
**self.cross_config
)
with open(crossfile, 'w') as outfile:
outfile.write(content)
return crossfile
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CFLAGS']
env['CXXFLAGS'] = " -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions --param=ssp-buffer-size=4 "+env['CXXFLAGS']
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
env['QEMU_LD_PREFIX'] = pj(self.root_path, "arm-linux-gnueabihf", "libc")
env['QEMU_SET_ENV'] = "LD_LIBRARY_PATH={}".format(
':'.join([
pj(self.root_path, "arm-linux-gnueabihf", "lib"),
pj(self.buildEnv.install_dir, 'lib'),
pj(self.buildEnv.install_dir, self.buildEnv.libprefix)
]))
def _download_zim(self, context):
zim_url = self.buildEnv.options.zim_file_url
if zim_url is None:
raise SkipCommand()
with open(
pj(self.source_path, self.target.custom_name,
'info.json')) as f:
app_info = json.load(f)
zim_url = app_info.get('zim_url', zim_url)
out_filename = urlparse(zim_url).path
out_filename = os.path.basename(out_filename)
zim_file = Remotefile(out_filename, '', zim_url)
self.buildEnv.download(zim_file)
shutil.copy(pj(self.buildEnv.archive_dir, out_filename),
pj(self.custom_build_path, app_info['zim_file']))
def _fix_permission_right(self, context):
context.try_skip(self.build_path)
bin_dirs = [pj(self.install_path, 'bin'),
pj(self.install_path, self.target.arch_full, 'bin'),
pj(self.install_path, 'libexec', 'gcc', self.target.arch_full, self.target.gccver)
]
for root, dirs, files in os.walk(self.install_path):
if not root in bin_dirs:
continue
for file_ in files:
file_path = pj(root, file_)
if os.path.islink(file_path):
continue
add_execution_right(file_path)
def _extract(self, context):
context.try_skip(self.extract_path)
if os.path.exists(self.extract_path):
shutil.rmtree(self.extract_path)
extract_archive(pj(self.buildEnv.archive_dir, self.archive.name),
self.buildEnv.source_dir,
topdir=self.archive_top_dir,
name=self.source_dir)
def _build_platform(self, context):
context.try_skip(self.install_path)
tools_dir = pj(self.install_path, 'tools')
shutil.copytree(self.source_path, tools_dir)
script = pj(tools_dir, 'android')
command = '{script} --verbose update sdk -a --no-ui --filter {packages}'
command = command.format(
script=script,
packages = ','.join(str(i) for i in [1,2,8,34,162])
)
# packages correspond to :
# - 1 : Android SDK Tools, revision 25.2.5
# - 2 : Android SDK Platform-tools, revision 25.0.3
# - 8 : Android SDK Build-tools, revision 24.0.1
# - 34 : SDK Platform Android 7.0, API 24, revision 2
# - 162 : Android Support Repository, revision 44
self.buildEnv.run_command(command, self.install_path, context, input="y\n")
def install_packages(self):
autoskip_file = pj(self.build_dir, ".install_packages_ok")
if self.distname in ('fedora', 'redhat', 'centos'):
package_installer = 'sudo dnf install {}'
package_checker = 'rpm -q --quiet {}'
elif self.distname in ('debian', 'Ubuntu'):
package_installer = 'sudo apt-get install {}'
package_checker = 'LANG=C dpkg -s {} 2>&1 | grep Status | grep "ok installed" 1>/dev/null 2>&1'
elif self.distname == 'Darwin':
package_installer = 'brew install {}'
package_checker = 'brew list -1 | grep -q {}'
mapper_name = "{host}_{target}".format(
host=self.distname,
target=self.platform_info)
try:
package_name_mapper = PACKAGE_NAME_MAPPERS[mapper_name]
except KeyError:
print("SKIP : We don't know which packages we must install to compile"
" a {target} {build_type} version on a {host} host.".format(
target=self.platform_info,
host=self.distname))
return
packages_list = package_name_mapper.get('COMMON', [])
for dep in self.targetsDict.values():
packages = package_name_mapper.get(dep.name)
if packages:
packages_list += packages
dep.skip = True
for dep in self.targetsDict.values():
packages = getattr(dep, 'extra_packages', [])
for package in packages:
packages_list += package_name_mapper.get(package, [])
if os.path.exists(autoskip_file):
print("SKIP")
return
packages_to_install = []
for package in packages_list:
print(" - {} : ".format(package), end="")
command = package_checker.format(package)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError:
print("NEEDED")
packages_to_install.append(package)
else:
print("SKIP")
if packages_to_install:
command = package_installer.format(" ".join(packages_to_install))
print(command)
subprocess.check_call(command, shell=True)
else:
print("SKIP, No package to install.")
with open(autoskip_file, 'w'):
pass
def configure_option(self):
options = "-DINSTALL_PKGCONFIG_DIR={}".format(
pj(self.buildEnv.install_dir, self.buildEnv.libprefix,
'pkgconfig'))
if self.buildEnv.platform_info.static:
options += " -DBUILD_SHARED_LIBS=false"
else:
options += " -DBUILD_SHARED_LIBS=true"
return options
def clean_intermediate_directories(self):
for subdir in os.listdir(self.build_dir):
subpath = pj(self.build_dir, subdir)
if subpath == self.install_dir:
continue
if os.path.isdir(subpath):
shutil.rmtree(subpath)
else:
os.remove(subpath)
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['CFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CFLAGS']
env['CXXFLAGS'] = '-fPIC -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 --sysroot={} '.format(self.root_path) + env['CXXFLAGS']
env['LDFLAGS'] = '--sysroot={} '.format(self.root_path) + env['LDFLAGS']
#env['CFLAGS'] = ' -fPIC -D_FILE_OFFSET_BITS=64 -O3 '+env['CFLAGS']
#env['CXXFLAGS'] = (' -D__OPTIMIZE__ -fno-strict-aliasing '
# ' -DU_HAVE_NL_LANGINFO_CODESET=0 '
# '-DU_STATIC_IMPLEMENTATION -O3 '
# '-DU_HAVE_STD_STRING -DU_TIMEZONE=0 ')+env['CXXFLAGS']
env['NDK_DEBUG'] = '0'
def _configure(self, context):
context.try_skip(self.build_path)
command = "{configure_script} {configure_option} --prefix {install_dir} --libdir {libdir}"
command = command.format(configure_script=pj(self.source_path,
self.configure_script),
configure_option=self.all_configure_option,
install_dir=self.buildEnv.install_dir,
libdir=pj(self.buildEnv.install_dir,
self.buildEnv.libprefix))
env = Defaultdict(str, os.environ)
if self.buildEnv.platform_info.static:
env['CFLAGS'] = env['CFLAGS'] + ' -fPIC'
if self.configure_env:
for k in self.configure_env:
if k.startswith('_format_'):
v = self.configure_env.pop(k)
v = v.format(buildEnv=self.buildEnv, env=env)
self.configure_env[k[8:]] = v
env.update(self.configure_env)
self.buildEnv.run_command(command, self.build_path, context, env=env)
def _build_platform(self, context):
context.try_skip(self.build_path)
script = pj(self.source_path, 'build/tools/make_standalone_toolchain.py')
add_execution_right(script)
command = '{script} --arch={arch} --api={api} --install-dir={install_dir} --force'
command = command.format(
script=script,
arch=self.target.arch,
api=self.target.api,
install_dir=self.install_path
)
self.buildEnv.run_command(command, self.build_path, context)
def binaries(self):
binaries = ((k,'{}-{}'.format(self.arch_full, v))
for k, v in (('CC', 'gcc'),
('CXX', 'g++'),
('AR', 'ar'),
('STRIP', 'strip'),
('WINDRES', 'windres'),
('RANLIB', 'ranlib'),
('LD', 'ld'))
)
return {k:pj(self.root_path, 'bin', v)
for k,v in binaries}
def gradle_option(self):
template = ("-i -P customDir={customDir}"
" -P zim_file_size={zim_size}"
" -P version_code={version_code}"
" -P version_name={version_name}"
" -P content_version_code={content_version_code}")
return template.format(
customDir=pj(self.build_path, 'custom'),
zim_size=self._get_zim_size(),
version_code=os.environ['VERSION_CODE'],
version_name=os.environ['VERSION_NAME'],
content_version_code=os.environ['CONTENT_VERSION_CODE'])
def __init__(self, options, targetsDict):
self.source_dir = pj(options.working_dir, "SOURCE")
build_dir = "BUILD_{}".format(options.target_platform)
self.build_dir = pj(options.working_dir, build_dir)
self.archive_dir = pj(options.working_dir, "ARCHIVE")
self.toolchain_dir = pj(options.working_dir, "TOOLCHAINS")
self.log_dir = pj(self.build_dir, 'LOGS')
self.install_dir = pj(self.build_dir, "INSTALL")
for d in (self.source_dir,
self.build_dir,
self.archive_dir,
self.toolchain_dir,
self.log_dir,
self.install_dir):
os.makedirs(d, exist_ok=True)
self.detect_platform()
self.ninja_command = self._detect_ninja()
if not self.ninja_command:
sys.exit("ERROR: ninja command not found")
self.meson_command = self._detect_meson()
if not self.meson_command:
sys.exit("ERROR: meson command not fount")
self.mesontest_command = "mesontest"
self.setup_build(options.target_platform)
self.setup_toolchains()
self.options = options
self.libprefix = options.libprefix or self._detect_libdir()
self.targetsDict = targetsDict
def _set_env(self, env, cross_compile_env, cross_compile_compiler, cross_compile_path):
if env is None:
env = Defaultdict(str, os.environ)
bin_dirs = []
if cross_compile_env:
for k, v in self.cross_config.get('env', {}).items():
if k.startswith('_format_'):
v = v.format(**self.cross_config)
k = k[8:]
env[k] = v
for toolchain in self.toolchains:
toolchain.set_env(env)
if cross_compile_compiler:
for toolchain in self.toolchains:
toolchain.set_compiler(env)
if cross_compile_path:
for tlc in self.toolchains:
bin_dirs += tlc.get_bin_dir()
pkgconfig_path = pj(self.install_dir, self.libprefix, 'pkgconfig')
env['PKG_CONFIG_PATH'] = ':'.join([env['PKG_CONFIG_PATH'], pkgconfig_path])
# Add ccache path
for p in ('/usr/lib/ccache', '/usr/lib64/ccache'):
if os.path.isdir(p):
ccache_path = [p]
break
else:
ccache_path = []
env['PATH'] = ':'.join(bin_dirs +
[pj(self.install_dir, 'bin')] +
ccache_path +
[env['PATH']])
env['LD_LIBRARY_PATH'] = ':'.join([env['LD_LIBRARY_PATH'],
pj(self.install_dir, 'lib'),
pj(self.install_dir, self.libprefix)
])
env['CPPFLAGS'] = " ".join(['-I'+pj(self.install_dir, 'include'), env['CPPFLAGS']])
env['LDFLAGS'] = " ".join(['-L'+pj(self.install_dir, 'lib'),
'-L'+pj(self.install_dir, self.libprefix),
env['LDFLAGS']])
return env
def command(self, name, function, *args):
print(" {} {} : ".format(name, self.name), end="", flush=True)
log = pj(self._log_dir, 'cmd_{}_{}.log'.format(name, self.name))
context = Context(name, log, True)
try:
ret = function(*args, context=context)
context._finalise()
print("OK")
return ret
except SkipCommand:
print("SKIP")
except subprocess.CalledProcessError:
print("ERROR")
try:
with open(log, 'r') as f:
print(f.read())
except:
pass
raise StopBuild()
except:
print("ERROR")
raise
def _configure(self, context):
if not os.path.exists(self.build_path):
shutil.copytree(self.source_path,
self.build_path,
symlinks=True)
try:
shutil.rmtree(pj(self.build_path, 'kiwixlib', 'src', 'main'))
except FileNotFoundError:
pass
shutil.copytree(pj(self.buildEnv.install_dir, 'kiwix-lib'),
pj(self.build_path, 'kiwixlib', 'src', 'main'))
os.makedirs(pj(self.build_path, 'app', 'src', 'main', 'assets',
'icu'),
exist_ok=True)
shutil.copy2(
pj(self.buildEnv.install_dir, 'share', 'icu', '58.2',
'icudt58l.dat'),
pj(self.build_path, 'app', 'src', 'main', 'assets', 'icu',
'icudt58l.dat'))
def source_path(self):
return pj(self.buildEnv.source_dir, self.source.source_dir)
def set_env(self, env):
env['PKG_CONFIG_LIBDIR'] = pj(self.root_path, 'lib', 'pkgconfig')
env['LIBS'] = " ".join(self.buildEnv.cross_config['extra_libs']) + " " +env['LIBS']
def get_bin_dir(self):
return [pj(self.root_path, 'bin')]
def root_path(self):
return pj(self.source_path, 'arm-bcm2708', 'gcc-linaro-arm-linux-gnueabihf-raspbian-x64')
def _fix_licenses(self, context):
context.try_skip(self.install_path)
os.makedirs(pj(self.install_path, 'licenses'), exist_ok=True)
with open(pj(self.install_path, 'licenses', 'android-sdk-license'), 'w') as f:
f.write("\n8933bad161af4178b1185d1a37fbf41ea5269c55\nd56f5187479451eabf01fb78af6dfcb131a6481e")
def root_path(self):
return pj(self.builder.install_path, 'sysroot')
def install_path(self):
return pj(self.buildEnv.toolchain_dir, self.target.full_name)
def get_bin_dir(self):
return [pj(self.builder.install_path, 'bin')]
