def build_arch(self, arch):
super(MobileInsightRecipe, self).build_arch(arch)
env = self.get_recipe_env(arch)
# self.build_cython_components(arch)
with current_directory(self.get_build_dir(arch.arch)):
hostpython = sh.Command(self.ctx.hostpython)
app_mk = join(self.get_build_dir(arch.arch), 'Application.mk')
app_setup = join(self.get_build_dir(arch.arch), 'setup.py')
if not exists(app_mk):
shprint(sh.cp, join(self.get_recipe_dir(), 'Application.mk'),
app_mk)
if not exists(app_setup):
shprint(sh.cp, join(self.get_recipe_dir(), 'setup.py'),
app_setup)
shprint(hostpython,
'setup.py',
'build_ext',
'-v',
_env=env,
_tail=10,
_critical=True)
shprint(hostpython,
'setup.py',
'install',
'-O2',
_env=env,
_tail=10,
_critical=True)
build_lib = glob.glob('./build/lib*')
assert len(build_lib) == 1
warning('MobileInsight -- stripping mobileinsight')
shprint(sh.find,
build_lib[0],
'-name',
'*.so',
'-exec',
env['STRIP'],
'{}',
';',
_tail=20,
_critical=True)
try:
warning('Copying GNU STL shared lib to {libs_dir}/{arch}'.format(
libs_dir=self.ctx.libs_dir, arch=arch))
shprint(
sh.cp,
'{ndk_dir}/sources/cxx-stl/gnu-libstdc++/{toolchain_version}/libs/{arch}/libgnustl_shared.so'
.format(ndk_dir=self.ctx.ndk_dir,
toolchain_version=self.toolchain_version,
arch=arch),
'{libs_dir}/{arch}'.format(libs_dir=self.ctx.libs_dir,
arch=arch))
except:
warning('Failed to copy GNU STL shared lib!')
def ledger_with_contents(contents):
return sh.Command('/usr/bin/ledger').bake(no_color=True,
file="-",
_in=contents)
def prepare_build_environment(self, user_sdk_dir, user_ndk_dir,
user_android_api, user_ndk_ver,
user_ndk_api):
'''Checks that build dependencies exist and sets internal variables
for the Android SDK etc.
..warning:: This *must* be called before trying any build stuff
'''
self.ensure_dirs()
if self._build_env_prepared:
return
ok = True
# Work out where the Android SDK is
sdk_dir = None
if user_sdk_dir:
sdk_dir = user_sdk_dir
# This is the old P4A-specific var
if sdk_dir is None:
sdk_dir = environ.get('ANDROIDSDK', None)
# This seems used more conventionally
if sdk_dir is None:
sdk_dir = environ.get('ANDROID_HOME', None)
# Checks in the buildozer SDK dir, useful for debug tests of p4a
if sdk_dir is None:
possible_dirs = glob.glob(
expanduser(
join('~', '.buildozer', 'android', 'platform',
'android-sdk-*')))
possible_dirs = [
d for d in possible_dirs
if not (d.endswith('.bz2') or d.endswith('.gz'))
]
if possible_dirs:
info('Found possible SDK dirs in buildozer dir: {}'.format(
', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
info('Will attempt to use SDK at {}'.format(possible_dirs[0]))
warning('This SDK lookup is intended for debug only, if you '
'use python-for-android much you should probably '
'maintain your own SDK download.')
sdk_dir = possible_dirs[0]
if sdk_dir is None:
raise BuildInterruptingException(
'Android SDK dir was not specified, exiting.')
self.sdk_dir = realpath(sdk_dir)
# Check what Android API we're using
android_api = None
if user_android_api:
android_api = user_android_api
info('Getting Android API version from user argument: {}'.format(
android_api))
elif 'ANDROIDAPI' in environ:
android_api = environ['ANDROIDAPI']
info('Found Android API target in $ANDROIDAPI: {}'.format(
android_api))
else:
info('Android API target was not set manually, using '
'the default of {}'.format(DEFAULT_ANDROID_API))
android_api = DEFAULT_ANDROID_API
android_api = int(android_api)
self.android_api = android_api
if self.android_api >= 21 and self.archs[0].arch == 'armeabi':
raise BuildInterruptingException(
'Asked to build for armeabi architecture with API '
'{}, but API 21 or greater does not support armeabi'.format(
self.android_api),
instructions=
'You probably want to build with --arch=armeabi-v7a instead')
if exists(join(sdk_dir, 'tools', 'bin', 'avdmanager')):
avdmanager = sh.Command(join(sdk_dir, 'tools', 'bin',
'avdmanager'))
targets = avdmanager('list',
'target').stdout.decode('utf-8').split('\n')
elif exists(join(sdk_dir, 'tools', 'android')):
android = sh.Command(join(sdk_dir, 'tools', 'android'))
targets = android('list').stdout.decode('utf-8').split('\n')
else:
raise BuildInterruptingException(
'Could not find `android` or `sdkmanager` binaries in Android SDK',
instructions='Make sure the path to the Android SDK is correct'
)
apis = [s for s in targets if re.match(r'^ *API level: ', s)]
apis = [re.findall(r'[0-9]+', s) for s in apis]
apis = [int(s[0]) for s in apis if s]
info('Available Android APIs are ({})'.format(', '.join(map(str,
apis))))
if android_api in apis:
info(('Requested API target {} is available, '
'continuing.').format(android_api))
else:
raise BuildInterruptingException(
('Requested API target {} is not available, install '
'it with the SDK android tool.').format(android_api))
# Find the Android NDK
# Could also use ANDROID_NDK, but doesn't look like many tools use this
ndk_dir = None
if user_ndk_dir:
ndk_dir = user_ndk_dir
info('Getting NDK dir from from user argument')
if ndk_dir is None: # The old P4A-specific dir
ndk_dir = environ.get('ANDROIDNDK', None)
if ndk_dir is not None:
info('Found NDK dir in $ANDROIDNDK: {}'.format(ndk_dir))
if ndk_dir is None: # Apparently the most common convention
ndk_dir = environ.get('NDK_HOME', None)
if ndk_dir is not None:
info('Found NDK dir in $NDK_HOME: {}'.format(ndk_dir))
if ndk_dir is None: # Another convention (with maven?)
ndk_dir = environ.get('ANDROID_NDK_HOME', None)
if ndk_dir is not None:
info('Found NDK dir in $ANDROID_NDK_HOME: {}'.format(ndk_dir))
if ndk_dir is None: # Checks in the buildozer NDK dir, useful
# # for debug tests of p4a
possible_dirs = glob.glob(
expanduser(
join('~', '.buildozer', 'android', 'platform',
'android-ndk-r*')))
if possible_dirs:
info('Found possible NDK dirs in buildozer dir: {}'.format(
', '.join([d.split(os.sep)[-1] for d in possible_dirs])))
info('Will attempt to use NDK at {}'.format(possible_dirs[0]))
warning('This NDK lookup is intended for debug only, if you '
'use python-for-android much you should probably '
'maintain your own NDK download.')
ndk_dir = possible_dirs[0]
if ndk_dir is None:
raise BuildInterruptingException(
'Android NDK dir was not specified')
self.ndk_dir = realpath(ndk_dir)
# Find the NDK version, and check it against what the NDK dir
# seems to report
ndk_ver = None
if user_ndk_ver:
ndk_ver = user_ndk_ver
if ndk_dir is not None:
info('Got NDK version from from user argument: {}'.format(
ndk_ver))
if ndk_ver is None:
ndk_ver = environ.get('ANDROIDNDKVER', None)
if ndk_ver is not None:
info('Got NDK version from $ANDROIDNDKVER: {}'.format(ndk_ver))
self.ndk = 'google'
try:
with open(join(ndk_dir, 'RELEASE.TXT')) as fileh:
reported_ndk_ver = fileh.read().split(' ')[0].strip()
except IOError:
pass
else:
if reported_ndk_ver.startswith('crystax-ndk-'):
reported_ndk_ver = reported_ndk_ver[12:]
self.ndk = 'crystax'
if ndk_ver is None:
ndk_ver = reported_ndk_ver
info(('Got Android NDK version from the NDK dir: {}'
).format(ndk_ver))
else:
if ndk_ver != reported_ndk_ver:
warning('NDK version was set as {}, but checking '
'the NDK dir claims it is {}.'.format(
ndk_ver, reported_ndk_ver))
warning('The build will try to continue, but it may '
'fail and you should check '
'that your setting is correct.')
warning('If the NDK dir result is correct, you don\'t '
'need to manually set the NDK ver.')
if ndk_ver is None:
warning('Android NDK version could not be found. This probably'
'won\'t cause any problems, but if necessary you can'
'set it with `--ndk-version=...`.')
self.ndk_ver = ndk_ver
ndk_api = None
if user_ndk_api:
ndk_api = user_ndk_api
info(
'Getting NDK API version (i.e. minimum supported API) from user argument'
)
elif 'NDKAPI' in environ:
ndk_api = environ.get('NDKAPI', None)
info('Found Android API target in $NDKAPI')
else:
ndk_api = min(self.android_api, DEFAULT_NDK_API)
warning(
'NDK API target was not set manually, using '
'the default of {} = min(android-api={}, default ndk-api={})'.
format(ndk_api, self.android_api, DEFAULT_NDK_API))
ndk_api = int(ndk_api)
self.ndk_api = ndk_api
if self.ndk_api > self.android_api:
raise BuildInterruptingException(
'Target NDK API is {}, higher than the target Android API {}.'.
format(self.ndk_api, self.android_api),
instructions=
('The NDK API is a minimum supported API number and must be lower '
'than the target Android API'))
info('Using {} NDK {}'.format(self.ndk.capitalize(), self.ndk_ver))
virtualenv = None
if virtualenv is None:
virtualenv = sh.which('virtualenv2')
if virtualenv is None:
virtualenv = sh.which('virtualenv-2.7')
if virtualenv is None:
virtualenv = sh.which('virtualenv')
if virtualenv is None:
raise IOError('Couldn\'t find a virtualenv executable, '
'you must install this to use p4a.')
self.virtualenv = virtualenv
info('Found virtualenv at {}'.format(virtualenv))
# path to some tools
self.ccache = sh.which("ccache")
if not self.ccache:
info('ccache is missing, the build will not be optimized in the '
'future.')
for cython_fn in ("cython", "cython3", "cython2", "cython-2.7"):
cython = sh.which(cython_fn)
if cython:
self.cython = cython
break
else:
raise BuildInterruptingException('No cython binary found.')
if not self.cython:
ok = False
warning("Missing requirement: cython is not installed")
# This would need to be changed if supporting multiarch APKs
arch = self.archs[0]
platform_dir = arch.platform_dir
toolchain_prefix = arch.toolchain_prefix
toolchain_version = None
self.ndk_platform = join(self.ndk_dir, 'platforms',
'android-{}'.format(self.ndk_api),
platform_dir)
if not exists(self.ndk_platform):
warning('ndk_platform doesn\'t exist: {}'.format(
self.ndk_platform))
ok = False
py_platform = sys.platform
if py_platform in ['linux2', 'linux3']:
py_platform = 'linux'
toolchain_versions = []
toolchain_path = join(self.ndk_dir, 'toolchains')
if isdir(toolchain_path):
toolchain_contents = glob.glob('{}/{}-*'.format(
toolchain_path, toolchain_prefix))
toolchain_versions = [
split(path)[-1][len(toolchain_prefix) + 1:]
for path in toolchain_contents
]
else:
warning('Could not find toolchain subdirectory!')
ok = False
toolchain_versions.sort()
toolchain_versions_gcc = []
for toolchain_version in toolchain_versions:
if toolchain_version[0].isdigit():
# GCC toolchains begin with a number
toolchain_versions_gcc.append(toolchain_version)
if toolchain_versions:
info('Found the following toolchain versions: {}'.format(
toolchain_versions))
info('Picking the latest gcc toolchain, here {}'.format(
toolchain_versions_gcc[-1]))
toolchain_version = toolchain_versions_gcc[-1]
else:
warning('Could not find any toolchain for {}!'.format(
toolchain_prefix))
ok = False
self.toolchain_prefix = toolchain_prefix
self.toolchain_version = toolchain_version
# Modify the path so that sh finds modules appropriately
environ['PATH'] = (
'{ndk_dir}/toolchains/{toolchain_prefix}-{toolchain_version}/'
'prebuilt/{py_platform}-x86/bin/:{ndk_dir}/toolchains/'
'{toolchain_prefix}-{toolchain_version}/prebuilt/'
'{py_platform}-x86_64/bin/:{ndk_dir}:{sdk_dir}/'
'tools:{path}').format(sdk_dir=self.sdk_dir,
ndk_dir=self.ndk_dir,
toolchain_prefix=toolchain_prefix,
toolchain_version=toolchain_version,
py_platform=py_platform,
path=environ.get('PATH'))
for executable in ("pkg-config", "autoconf", "automake", "libtoolize",
"tar", "bzip2", "unzip", "make", "gcc", "g++"):
if not sh.which(executable):
warning("Missing executable: {} is not installed".format(
executable))
if not ok:
raise BuildInterruptingException(
'python-for-android cannot continue due to the missing executables above'
)
def copylibs_function(soname, objs_paths, extra_link_dirs=[], env=None):
print('objs_paths are', objs_paths)
re_needso = re.compile(
r'^.*\(NEEDED\)\s+Shared library: \[lib(.*)\.so\]\s*$')
blacklist_libs = (
'c',
'stdc++',
'dl',
'python2.7',
'sdl',
'sdl_image',
'sdl_ttf',
'z',
'm',
'GLESv2',
'jpeg',
'png',
'log',
# bootstrap takes care of sdl2 libs (if applicable)
'SDL2',
'SDL2_ttf',
'SDL2_image',
'SDL2_mixer',
)
found_libs = []
sofiles = []
if env and 'READELF' in env:
readelf = env['READELF']
elif 'READELF' in os.environ:
readelf = os.environ['READELF']
else:
readelf = sh.which('readelf').strip()
readelf = sh.Command(readelf).bake('-d')
dest = dirname(soname)
for directory in objs_paths:
for fn in os.listdir(directory):
fn = join(directory, fn)
if not fn.endswith('.libs'):
continue
dirfn = fn[:-1] + 'dirs'
if not exists(dirfn):
continue
with open(fn) as f:
libs = f.read().strip().split(' ')
needed_libs = [
lib for lib in libs if lib and lib not in blacklist_libs
and lib not in found_libs
]
while needed_libs:
print('need libs:\n\t' + '\n\t'.join(needed_libs))
start_needed_libs = needed_libs[:]
found_sofiles = []
with open(dirfn) as f:
libdirs = f.read().split()
for libdir in libdirs:
if not needed_libs:
break
if libdir == dest:
# don't need to copy from dest to dest!
continue
libdir = libdir.strip()
print('scanning', libdir)
for lib in needed_libs[:]:
if lib in found_libs:
continue
if lib.endswith('.a'):
needed_libs.remove(lib)
found_libs.append(lib)
continue
lib_a = 'lib' + lib + '.a'
libpath_a = join(libdir, lib_a)
lib_so = 'lib' + lib + '.so'
libpath_so = join(libdir, lib_so)
plain_so = lib + '.so'
plainpath_so = join(libdir, plain_so)
sopath = None
if exists(libpath_so):
sopath = libpath_so
elif exists(plainpath_so):
sopath = plainpath_so
if sopath:
print('found', lib, 'in', libdir)
found_sofiles.append(sopath)
needed_libs.remove(lib)
found_libs.append(lib)
continue
if exists(libpath_a):
print('found', lib, '(static) in', libdir)
needed_libs.remove(lib)
found_libs.append(lib)
continue
for sofile in found_sofiles:
print('scanning dependencies for', sofile)
out = readelf(sofile)
for line in out.splitlines():
needso = re_needso.match(line)
if needso:
lib = needso.group(1)
if (lib not in needed_libs
and lib not in found_libs
and lib not in blacklist_libs):
needed_libs.append(needso.group(1))
sofiles += found_sofiles
if needed_libs == start_needed_libs:
raise RuntimeError(
'Failed to locate needed libraries!\n\t' +
'\n\t'.join(needed_libs))
print('Copying libraries')
for lib in sofiles:
shprint(sh.cp, lib, dest)
def prepare_commands():
notify_send_cmd = sh.Command("notify-send")
notify_send_cmd = notify_send_cmd.bake("--icon==gtk-info")
notmuch_cmd = sh.Command("notmuch")
return notify_send_cmd, notmuch_cmd
from __future__ import unicode_literals
from collections import namedtuple
from json import dumps
import sh
try:
pip = sh.Command('pip')
PIP = True
except sh.CommandNotFound:
PIP = False
class PIPNotFound(Exception):
pass
class PropertyNamespace(object):
_registry = {}
@classmethod
def get(cls, name):
return cls._registry[name]
@classmethod
def get_all(cls):
return cls._registry.values()
def __init__(self, name, label):
self.name = name
app.config['GIT_EXECUTABLE'] = 'git'
try:
app.config.from_pyfile(
os.path.join(app.config.get('CONTENT_DIR'), 'config.py'))
except IOError:
print(
"Startup Failure: You need to place a "
"config.py in your content directory.")
manager = Manager(app)
loginmanager = LoginManager()
loginmanager.init_app(app)
loginmanager.login_view = 'user_login'
git = sh.Command(
app.config.get('GIT_EXECUTABLE')).bake(_cwd=app.config.get('CONTENT_DIR'))
"""
Wiki classes
~~~~~~~~~~~~
"""
class Processors(object):
"""This class is collection of processors for various content items.
"""
def __init__(self, content=""):
"""Initialization function. Runs Processors().pre() on content.
Args:
None
def run_distribute(self):
info_main(
'# Creating Android project from build and {} bootstrap'.format(
self.name))
info(
'This currently just copies the build stuff straight from the build dir.'
)
shprint(sh.rm, '-rf', self.dist_dir)
shprint(sh.cp, '-r', self.build_dir, self.dist_dir)
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
fileh.write('ndk.dir={}'.format(self.ctx.ndk_dir))
arch = self.ctx.archs[0]
if len(self.ctx.archs) > 1:
raise ValueError(
'built for more than one arch, but bootstrap cannot handle that yet'
)
info('Bootstrap running with arch {}'.format(arch))
with current_directory(self.dist_dir):
info('Copying python distribution')
if not exists(
'private') and not self.ctx.python_recipe.from_crystax:
shprint(sh.mkdir, 'private')
if not exists(
'crystax_python') and self.ctx.python_recipe.from_crystax:
shprint(sh.mkdir, 'crystax_python')
shprint(sh.mkdir, 'crystax_python/crystax_python')
if not exists('assets'):
shprint(sh.mkdir, 'assets')
hostpython = sh.Command(self.ctx.hostpython)
if not self.ctx.python_recipe.from_crystax:
try:
shprint(hostpython,
'-OO',
'-m',
'compileall',
self.ctx.get_python_install_dir(),
_tail=10,
_filterout="^Listing")
except sh.ErrorReturnCode:
pass
if not exists('python-install'):
shprint(sh.cp, '-a', self.ctx.get_python_install_dir(),
'./python-install')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_aars(arch)
self.distribute_javaclasses(self.ctx.javaclass_dir)
if not self.ctx.python_recipe.from_crystax:
info('Filling private directory')
if not exists(join('private', 'lib')):
info('private/lib does not exist, making')
shprint(sh.cp, '-a', join('python-install', 'lib'),
'private')
shprint(sh.mkdir, '-p', join('private', 'include',
'python2.7'))
if exists(join('libs', arch.arch, 'libpymodules.so')):
shprint(sh.mv, join('libs', arch.arch, 'libpymodules.so'),
'private/')
shprint(
sh.cp,
join('python-install', 'include',
'python2.7', 'pyconfig.h'),
join('private', 'include', 'python2.7/'))
info('Removing some unwanted files')
shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so'))
shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig'))
libdir = join(self.dist_dir, 'private', 'lib', 'python2.7')
site_packages_dir = join(libdir, 'site-packages')
with current_directory(libdir):
# shprint(sh.xargs, 'rm', sh.grep('-E', '*\.(py|pyx|so\.o|so\.a|so\.libs)$', sh.find('.')))
removes = []
for dirname, something, filens in walk('.'):
for filename in filens:
for suffix in ('py', 'pyc', 'so.o', 'so.a',
'so.libs'):
if filename.endswith(suffix):
removes.append(filename)
shprint(sh.rm, '-f', *removes)
info('Deleting some other stuff not used on android')
# To quote the original distribute.sh, 'well...'
# shprint(sh.rm, '-rf', 'ctypes')
shprint(sh.rm, '-rf', 'lib2to3')
shprint(sh.rm, '-rf', 'idlelib')
for filename in glob.glob('config/libpython*.a'):
shprint(sh.rm, '-f', filename)
shprint(sh.rm, '-rf', 'config/python.o')
# shprint(sh.rm, '-rf', 'lib-dynload/_ctypes_test.so')
# shprint(sh.rm, '-rf', 'lib-dynload/_testcapi.so')
else: # Python *is* loaded from crystax
ndk_dir = self.ctx.ndk_dir
py_recipe = self.ctx.python_recipe
python_dir = join(ndk_dir, 'sources', 'python',
py_recipe.version, 'libs', arch.arch)
shprint(sh.cp, '-r', join(python_dir, 'stdlib.zip'),
'crystax_python/crystax_python')
shprint(sh.cp, '-r', join(python_dir, 'modules'),
'crystax_python/crystax_python')
shprint(sh.cp, '-r', self.ctx.get_python_install_dir(),
'crystax_python/crystax_python/site-packages')
info('Renaming .so files to reflect cross-compile')
site_packages_dir = 'crystax_python/crystax_python/site-packages'
filens = shprint(
sh.find, site_packages_dir, '-iname',
'*.so').stdout.decode('utf-8').split('\n')[:-1]
for filen in filens:
parts = filen.split('.')
if len(parts) <= 2:
continue
shprint(sh.mv, filen, filen.split('.')[0] + '.so')
site_packages_dir = join(abspath(curdir), site_packages_dir)
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(ServiceOnlyBootstrap, self).run_distribute()
Copyright (c) 2021 ABLECLOUD Co. Ltd.
호스트의 디스크 목록을 조회하는 스크립트
최초작성일 : 2021-03-15
'''
import argparse
import json
import logging
from ablestack import *
import os
import sh
import distro
lsblk_cmd = sh.Command('/usr/bin/lsblk')
if distro.linux_distribution() == ('CentOS Linux', '8', ''):
# print('centos8')
lspci_cmd = sh.Command('/usr/sbin/lspci')
else:
# print('other')
lspci_cmd = sh.Command('/usr/bin/lspci')
env = os.environ.copy()
env['LANG'] = "en_US.utf-8"
env['LANGUAGE'] = "en"
"""
입력된 argument를 파싱하여 dictionary 처럼 사용하게 만들어 주는 parser를 생성하는 함수
:return: argparse.ArgumentParser
"""
def __init__(self, preprocess_task):
super().__init__()
self.preprocess_task = preprocess_task
self.prodigal = sh.Command(self.config['prodigal']['bin']).bake(
i=self.preprocess_task.output(), a=self.output())
def build_arch(self, arch):
with current_directory(self.get_build_dir(arch.arch)):
env = arch.get_env()
flags = ['--disable-everything']
cflags = []
ldflags = []
if 'openssl' in self.ctx.recipe_build_order:
flags += [
'--enable-openssl',
'--enable-nonfree',
'--enable-protocol=https,tls_openssl',
]
build_dir = Recipe.get_recipe(
'openssl', self.ctx).get_build_dir(arch.arch)
cflags += ['-I' + build_dir + '/include/']
ldflags += ['-L' + build_dir]
if 'ffpyplayer_codecs' in self.ctx.recipe_build_order:
# libx264
flags += ['--enable-libx264']
build_dir = Recipe.get_recipe(
'libx264', self.ctx).get_build_dir(arch.arch)
cflags += ['-I' + build_dir + '/include/']
ldflags += ['-lx264', '-L' + build_dir + '/lib/']
# libshine
flags += ['--enable-libshine']
build_dir = Recipe.get_recipe(
'libshine', self.ctx).get_build_dir(arch.arch)
cflags += ['-I' + build_dir + '/include/']
ldflags += ['-lshine', '-L' + build_dir + '/lib/']
# Enable all codecs:
flags += [
'--enable-parsers',
'--enable-decoders',
'--enable-encoders',
'--enable-muxers',
'--enable-demuxers',
]
else:
# Enable codecs only for .mp4:
flags += [
'--enable-parser=aac,ac3,h261,h264,mpegaudio,mpeg4video,mpegvideo,vc1',
'--enable-decoder=aac,h264,mpeg4,mpegvideo',
'--enable-muxer=h264,mov,mp4,mpeg2video',
'--enable-demuxer=aac,h264,m4v,mov,mpegvideo,vc1',
]
# needed to prevent _ffmpeg.so: version node not found for symbol av_init_packet@LIBAVFORMAT_52
# /usr/bin/ld: failed to set dynamic section sizes: Bad value
flags += [
'--disable-symver',
]
# disable binaries / doc
flags += [
'--disable-ffmpeg',
'--disable-ffplay',
'--disable-ffprobe',
'--disable-ffserver',
'--disable-doc',
]
# other flags:
flags += [
'--enable-filter=aresample,resample,crop,adelay,volume,scale',
'--enable-protocol=file,http',
'--enable-small',
'--enable-hwaccels',
'--enable-gpl',
'--enable-pic',
'--disable-static',
'--enable-shared',
]
if 'arm64' in arch.arch:
cross_prefix = 'aarch64-linux-android-'
arch_flag = 'aarch64'
else:
cross_prefix = 'arm-linux-androideabi-'
arch_flag = 'arm'
# android:
flags += [
'--target-os=android',
'--cross-prefix={}'.format(cross_prefix),
'--arch={}'.format(arch_flag),
'--sysroot=' + self.ctx.ndk_platform,
'--enable-neon',
'--prefix={}'.format(realpath('.')),
]
if arch_flag == 'arm':
cflags += [
'-mfpu=vfpv3-d16',
'-mfloat-abi=softfp',
'-fPIC',
]
env['CFLAGS'] += ' ' + ' '.join(cflags)
env['LDFLAGS'] += ' ' + ' '.join(ldflags)
configure = sh.Command('./configure')
shprint(configure, *flags, _env=env)
shprint(sh.make, '-j4', _env=env)
shprint(sh.make, 'install', _env=env)
# copy libs:
sh.cp('-a', sh.glob('./lib/lib*.so'),
self.ctx.get_libs_dir(arch.arch))
def main():
parser = argparse.ArgumentParser(
description=
"For the specified repository, assess its RAML and schema files using raml-cop."
)
parser.add_argument(
"-i",
"--input",
default=".",
help=
"Directory of the repo git clone. (Default: current working directory)"
)
parser.add_argument(
"-f",
"--file",
default="",
help=
"Limit to this particular pathname, e.g. ramls/item-storage.raml (Default: '' so all files)"
)
parser.add_argument(
"-v",
"--validate-only",
action="store_true",
help=
"Just assess the RAML files. No schema assessment. (Default: False)")
parser.add_argument(
"-j",
"--json-only",
action="store_true",
help=
"Just assess the JSON schema files. No RAML assessment. (Default: False)"
)
parser.add_argument(
"-l",
"--loglevel",
choices=["debug", "info", "warning", "error", "critical"],
default="info",
help="Logging level. (Default: warning)")
parser.add_argument(
"-d",
"--dev",
action="store_true",
help="Development mode. Use local config file. (Default: False)")
parser.add_argument(
"-c",
"--config",
default="api.yml",
help="Pathname to local configuration file. (Default: api.yml)")
args = parser.parse_args()
loglevel = LOGLEVELS.get(args.loglevel.lower(), logging.NOTSET)
# Need stdout to enable Jenkins to redirect into an output file
logging.basicConfig(stream=sys.stdout,
format="%(levelname)s: %(name)s: %(message)s",
level=loglevel)
logger1 = logging.getLogger("lint-raml")
logger2 = logging.getLogger("lint-raml-cop")
logger3 = logging.getLogger("lint-raml-schema")
logging.getLogger("sh").setLevel(logging.ERROR)
logging.getLogger("requests").setLevel(logging.ERROR)
# Display a version string
logger1.info("Using lint-raml version: %s", SCRIPT_VERSION)
# Process and validate the input parameters
if args.input.startswith("~"):
git_input_dir = os.path.expanduser(args.input)
else:
git_input_dir = args.input
if not os.path.exists(git_input_dir):
logger1.critical(
"Specified input directory of git clone (-i) not found: %s",
git_input_dir)
return 2
# Ensure that commands are available
if sh.which("jq"):
has_jq = True
else:
logger1.warning(
"'jq' is not available. So will not do extra JSON assessment.")
has_jq = False
bin_raml_cop = os.path.join(sys.path[0], "node_modules", ".bin",
"raml-cop")
if not os.path.exists(bin_raml_cop):
logger1.critical("'raml-cop' is not available.")
logger1.critical(
"Do 'yarn install' in folio-tools/lint-raml directory.")
return 2
# Get the repository name
try:
repo_url = sh.git.config("--get",
"remote.origin.url",
_cwd=git_input_dir).stdout.decode().strip()
except sh.ErrorReturnCode as err:
logger1.critical("Trouble doing 'git config': %s", err.stderr.decode())
logger1.critical(
"Could not determine remote.origin.url of git clone in specified input directory: %s",
git_input_dir)
return 2
else:
repo_name = os.path.splitext(os.path.basename(repo_url))[0]
if args.file:
specific_raml_file_pn = os.path.join(git_input_dir, args.file)
if not os.path.exists(specific_raml_file_pn):
logger1.critical("Specific RAML file '%s' does not exist in '%s'",
specific_raml_file_pn, repo_name)
logger1.critical(
"Needs to be pathname relative to top-level, e.g. ramls/item-storage.raml"
)
return 2
# Get the configuration metadata for all repositories that are known to have RAML
if args.config.startswith("~"):
config_local_pn = os.path.expanduser(args.config)
else:
config_local_pn = args.config
if args.dev is False:
try:
http_response = requests.get(CONFIG_FILE)
http_response.raise_for_status()
except requests.exceptions.HTTPError as err:
logger1.critical("HTTP error retrieving configuration file: %s",
err)
return 2
except Exception as err:
logger1.critical("Error retrieving configuration file: %s", err)
return 2
else:
try:
config = yaml.safe_load(http_response.text)
except yaml.YAMLError as err:
logger1.critical(
"Trouble parsing YAML configuration file '%s': %s",
CONFIG_FILE, err)
return 2
else:
if not os.path.exists(config_local_pn):
logger1.critical(
"Development mode specified (-d) but config file (-c) not found: %s",
config_local_pn)
return 2
with open(config_local_pn) as input_fh:
try:
config = yaml.safe_load(input_fh)
except yaml.YAMLError as err:
logger1.critical(
"Trouble parsing YAML configuration file '%s': %s",
config_local_pn, err)
return 2
if config is None:
logger1.critical("Configuration data was not loaded.")
return 2
if repo_name not in config:
logger1.warning("No configuration found for repository '%s'",
repo_name)
logger1.warning("See FOLIO-903. Add an entry to api.yml")
logger1.warning("Attempting default configuration.")
config[repo_name] = config["default"]
config[repo_name][0]["files"].remove("dummy")
# The yaml parser gags on the "!include".
# http://stackoverflow.com/questions/13280978/pyyaml-errors-on-in-a-string
yaml.add_constructor(u"!include",
construct_raml_include,
Loader=yaml.SafeLoader)
# Detect any schema $ref
schema_ref_re = re.compile(r'( +"\$ref"[ :]+")([^"]+)(".*)')
# Handle issue messages of parser
message_avoid_re = re.compile(r'^(\[[^]]+\]) ([^:]+):(.*)$')
# Process each configured set of RAML files
version_re = re.compile(r"^#%RAML ([0-9.]+)")
exit_code = 0 # Continue processing to detect various issues, then return the result.
input_dir = git_input_dir
for docset in config[repo_name]:
logger1.info("Investigating and determining configuration: %s",
os.path.join(repo_name, docset["directory"]))
ramls_dir = os.path.join(input_dir, docset["directory"])
logger1.debug("ramls_dir=%s", ramls_dir)
version_ramlutil_v1 = True
if not os.path.exists(ramls_dir):
logger1.warning("The specified 'ramls' directory not found: %s",
os.path.join(repo_name, docset["directory"]))
logger1.warning("See FOLIO-903. Update entry in api.yml")
logger1.warning("Attempting default.")
docset["directory"] = config["default"][0]["directory"]
ramls_dir = os.path.join(input_dir, docset["directory"])
if not os.path.exists(ramls_dir):
logger1.critical(
"The default 'ramls' directory not found: %s/%s",
repo_name, docset["directory"])
return 2
if docset["ramlutil"] is not None:
ramlutil_dir = os.path.join(input_dir, docset["ramlutil"])
if not os.path.exists(ramlutil_dir):
logger1.warning(
"The specified 'raml-util' directory not found: %s",
os.path.join(repo_name, docset["ramlutil"]))
logger1.warning("See FOLIO-903. Update entry in api.yml")
else:
# Detect if new raml-util
auth_trait_pn = os.path.join(input_dir, docset["ramlutil"],
"traits/auth.raml")
if os.path.exists(auth_trait_pn):
version_ramlutil_v1 = False
# If is using RMB, then there are various peculiarities to assess.
try:
is_rmb = docset["rmb"]
except KeyError:
is_rmb = True
# Some repos have no RAMLs.
# Currently this script is also processing schemas FOLIO-1447, so this can be intentional.
try:
is_schemas_only = docset["schemasOnly"]
except KeyError:
is_schemas_only = False
# Ensure configuration and find any RAML files not configured
configured_raml_files = []
try:
docset["files"]
except KeyError:
pass
else:
if isinstance(docset["files"], Iterable):
for raml_name in docset["files"]:
raml_fn = "{0}.raml".format(raml_name)
configured_raml_files.append(raml_fn)
exclude_list = [
"raml-util", "rtypes", "traits", "examples", "bindings",
"node_modules", ".git"
]
try:
exclude_list.extend(docset["excludes"])
except KeyError:
pass
excludes = set(exclude_list)
found_raml_files = []
raml_files = []
found_schema_files = []
if docset["label"] == "shared":
# If this is the top-level of the shared space, then do not descend
pattern = os.path.join(ramls_dir, "*.raml")
for raml_fn in glob.glob(pattern):
raml_pn = os.path.relpath(raml_fn, ramls_dir)
found_raml_files.append(raml_pn)
else:
for root, dirs, files in os.walk(ramls_dir, topdown=True):
dirs[:] = [d for d in dirs if d not in excludes]
for raml_fn in fnmatch.filter(files, "*.raml"):
raml_pn = os.path.relpath(os.path.join(root, raml_fn),
ramls_dir)
found_raml_files.append(raml_pn)
# Also find the JSON Schemas to later scan them
try:
schemas_dir = os.path.join(input_dir, docset["schemasDirectory"])
except KeyError:
schemas_dir = os.path.join(input_dir, docset["directory"])
else:
if not os.path.exists(schemas_dir):
logger1.warning(
"The specified 'schemasDirectory' not found: %s",
os.path.join(repo_name, docset["schemasDirectory"]))
logger1.warning("See FOLIO-903. Update entry in api.yml")
logger1.warning("Attempting default.")
schemas_dir = os.path.join(input_dir, docset["directory"])
if docset["label"] == "shared":
# If this is the top-level of the shared space, then do not descend
pattern = os.path.join(schemas_dir, "*.schema")
for schema_fn in glob.glob(pattern):
schema_pn = os.path.relpath(schema_fn, schemas_dir)
found_schema_files.append(schema_pn)
else:
for root, dirs, files in os.walk(schemas_dir, topdown=True):
dirs[:] = [d for d in dirs if d not in excludes]
logger1.debug("Looking for JSON schema files: %s", root)
for filename in files:
if filename.endswith((".json", ".schema")):
schema_pn = os.path.relpath(
os.path.join(root, filename), schemas_dir)
found_schema_files.append(schema_pn)
logger1.debug("found_schema_files: %s", found_schema_files)
for raml_fn in configured_raml_files:
if raml_fn not in found_raml_files:
logger1.warning("Configured file not found: %s", raml_fn)
logger1.warning(
"Configuration needs to be updated (FOLIO-903).")
else:
raml_files.append(raml_fn)
for raml_fn in found_raml_files:
if raml_fn not in configured_raml_files:
raml_files.append(raml_fn)
logger1.warning("Missing from configuration: %s", raml_fn)
logger1.warning(
"Configuration needs to be updated (FOLIO-903).")
logger1.debug("configured_raml_files: %s", configured_raml_files)
logger1.debug("found_raml_files: %s", found_raml_files)
logger1.debug("raml_files: %s", raml_files)
if found_schema_files:
if args.validate_only:
logger1.info(
"Not assessing schema descriptions, as per option '--validate-only'."
)
else:
issues_flag = assess_schema_descriptions(
schemas_dir, found_schema_files, has_jq)
if issues_flag:
exit_code = 1
if args.json_only:
logger1.info(
"Not assessing RAML/Schema or examples against schema, as per option '--json-only'."
)
continue
if not is_schemas_only:
logger1.info(
"Assessing RAML files (https://dev.folio.org/guides/raml-cop/):"
)
if not raml_files:
logger1.error("No RAML files found in %s", ramls_dir)
exit_code = 1
for raml_fn in sorted(raml_files):
if args.file:
if os.path.join(docset["directory"], raml_fn) != args.file:
logger1.info("Skipping RAML file: %s", raml_fn)
continue
input_pn = os.path.join(ramls_dir, raml_fn)
if not os.path.exists(input_pn):
logger1.warning("Missing configured input file '%s'",
os.path.join(repo_name, raml_fn))
logger1.warning(
"Configuration needs to be updated (FOLIO-903).")
continue
# Determine raml version
version_value = None
with open(input_pn, "r") as input_fh:
for num, line in enumerate(input_fh):
match = re.search(version_re, line)
if match:
version_value = match.group(1)
break
if not version_value:
logger1.error(
"Could not determine RAML version for file '%s' so skipping.",
raml_fn)
exit_code = 1
continue
logger2.info("Processing RAML v%s file: %s", version_value,
raml_fn)
if version_value != "0.8" and not version_ramlutil_v1:
logger1.error(
"The raml-util is not RAML-1.0 version. Update git submodule."
)
exit_code = 2
continue
# Now process this RAML file
# First load the content to extract some details.
(schemas,
issues_flag) = gather_declarations(input_pn, raml_fn,
version_value, is_rmb,
input_dir, docset["directory"])
logger1.debug("Found %s declared schemas or types files.",
len(schemas))
if issues_flag:
exit_code = 1
# Ensure each $ref referenced schema file exists, is useable, and is declared in the RAML
for schema in schemas:
schema_pn = os.path.normpath(
os.path.join(ramls_dir, schemas[schema]))
if not os.path.exists(schema_pn):
# Missing file was already reported
continue
schema_dir = os.path.dirname(schema_pn)
with open(schema_pn) as input_fh:
lines = list(input_fh)
for line in lines:
match = re.search(schema_ref_re, line)
if match:
ref_value = match.group(2)
logger1.debug(
"Found schema $ref '%s' in schema file '%s'",
ref_value, schemas[schema])
relative_schema_ref_fn = os.path.normpath(
os.path.join(os.path.dirname(schemas[schema]),
ref_value))
logger1.debug(" relative_schema_ref_fn=%s",
relative_schema_ref_fn)
relative_schema_ref_pn = os.path.normpath(
os.path.join(ramls_dir, relative_schema_ref_fn))
if not is_rmb:
logger1.debug(
"Not RMB type, so just report if file not found."
)
if not os.path.exists(relative_schema_ref_pn):
logger1.error("File not found: %s",
relative_schema_ref_pn)
logger1.error(
" via schema $ref '%s' in schema file '%s'",
ref_value, schemas[schema])
exit_code = 1
else:
if version_value != "0.8":
#logger1.debug("Is RMB >= v20 and 1.0, so report if file not found.")
if not os.path.exists(relative_schema_ref_pn):
logger1.error("File not found: %s",
relative_schema_ref_pn)
logger1.error(
" via schema $ref '%s' in schema file '%s'",
ref_value, schemas[schema])
exit_code = 1
else:
#logger1.debug("Is RMB < v20 and 0.8, so report if file not found, and ensure declaration.")
# RMB < v20 enables $ref in schema to be a pathname, if the position in the filesystem
# and its use in the RAML meets strict conditions.
if not os.path.exists(relative_schema_ref_pn):
logger1.error("File not found: %s",
relative_schema_ref_pn)
logger1.error(
" via schema $ref '%s' in schema file '%s'",
ref_value, schemas[schema])
exit_code = 1
else:
# This RMB version has an extra bit of weirdness.
# If the declaration of a schema key in the raml file needs to be a path,
# (e.g. in raml-util mod-users-bl.raml) then if its included schema has $ref
# to another schema using a relative path with dot-dots, then that schema's key
# needs to be adjusted according to the depth of the path in the top-level
# schema key (e.g. for $ref=../metadata.schema).
if "../" in ref_value:
rel_ref_value = ref_value
for x in range(0, schema.count("/")):
logger1.debug(
" dot-dot count x=%s",
x + 1)
rel_ref_value = re.sub(
"\.\./",
"",
rel_ref_value,
count=1)
logger1.debug(" rel_ref_value=%s",
rel_ref_value)
try:
schemas[rel_ref_value]
except KeyError:
logger1.error(
"The schema reference '%s' defined in '%s' needs to be declared as '%s' in RAML file.",
ref_value, schemas[schema],
rel_ref_value)
exit_code = 1
else:
try:
schemas[ref_value]
except KeyError:
logger1.error(
"The schema reference '%s' defined in '%s' is not declared in RAML file.",
ref_value, schemas[schema])
exit_code = 1
# Sool raml-cop onto it.
cmd_raml_cop = sh.Command(bin_raml_cop)
try:
cmd_raml_cop(input_pn, no_color=True)
except sh.ErrorReturnCode_1 as err:
(issues_list, errors_remain) = avoid_specific_errors(
repo_name,
err.stdout.decode().split(os.linesep), message_avoid_re)
if errors_remain:
logger2.error(" raml-cop detected errors with %s:\n%s",
raml_fn, '\n'.join(issues_list))
exit_code = 1
else:
logger2.warning(
" raml-cop detected warnings with %s:\n%s", raml_fn,
'\n'.join(issues_list))
exit_code = 0
else:
logger2.info(" raml-cop did not detect any errors with %s",
raml_fn)
# Report the outcome
if exit_code == 1:
logger1.error("There were processing errors. See list above.")
elif exit_code == 2:
logger1.error("There were processing errors. See list above.")
else:
logger1.info("Did not detect any errors.")
logging.shutdown()
return exit_code
def _run(self,
language,
target_dir,
browser,
skip_screenshots=True,
force=False,
screenshot_build="source",
target_version=None):
sphinx_build = sh.Command("sphinx-build")
# check if sources exist for this language
section = "manual-%s" % language
target_type = self.config.get(section, "target-type")
source_dir = os.path.join(self.root_dir,
self.config.get(section, "source"))
if target_dir is None:
target_dir = os.path.join(self.root_dir,
self.config.get(section, "target"))
else:
target_dir = os.path.join(self.root_dir, target_dir)
target_dir = target_dir.replace(
"<version>",
self._get_doc_target_path()
if target_version is None else target_version)
print("generating doc to %s" % target_dir)
if not os.path.exists(source_dir):
self.log.error(
"no sources found for manual (%s) in language '%s'" %
(source_dir, language))
sys.exit(1)
if force and os.path.exists(target_dir):
# delete target dir
print("deleting old content in '%s'" % target_dir)
shutil.rmtree(target_dir)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# first run generates the widget-example configs
print(
'================================================================================'
)
print('sphinx_build: first run')
print(
'================================================================================'
)
sphinx_build("-b",
target_type,
source_dir,
target_dir,
_out=self.process_output,
_err=self.process_output)
if not skip_screenshots:
grunt = sh.Command("grunt")
# generate the screenshots
grunt("--force",
"screenshots",
"--subDir=manual",
"--browserName=%s" % browser,
"--target=%s" % screenshot_build,
_out=self.process_output,
_err=self.process_output)
# 2dn run with access to the generated screenshots
print(
'================================================================================'
)
print('sphinx_build: second run')
print(
'================================================================================'
)
sphinx_build("-b",
target_type,
source_dir,
target_dir,
_out=self.process_output,
_err=self.process_output)
with open(os.path.join(target_dir, "..", "version"), "w+") as f:
f.write(self._get_source_version())
# create symlinks
symlinkname = ''
git = sh.Command("git")
branch = git("rev-parse", "--abbrev-ref", "HEAD").strip() if os.environ.get('TRAVIS_BRANCH') is None \
else os.environ.get('TRAVIS_BRANCH')
if branch == "develop":
# handle develop builds:
print('detected development build')
symlinkname = self.config.get("DEFAULT", "develop-version-mapping")
elif branch == "master":
# handle releases:
print('detected build of most recent version of master branch')
symlinkname = self.config.get("DEFAULT",
"most-recent-version-mapping")
if '' != symlinkname:
symlinktarget = os.path.join(target_dir, "..")
print("setting symlink '%s' to '%s'" %
(symlinkname, symlinktarget))
cwd = os.getcwd()
os.chdir(os.path.join(symlinktarget, ".."))
try:
os.remove(symlinkname)
except Exception as e:
print(str(e))
ls = sh.Command("ls")
print(ls("-la"))
os.symlink(os.path.relpath(symlinktarget), symlinkname)
os.chdir(cwd)
def run(self):
main = sh.Command(self.tmpdir + '/main')
loop_factor = self.benchmark_definition['loop_factor']
results = main(int(5000000 * loop_factor))
return parse_results(results.splitlines())
def run_distribute(self):
info_main("# Creating Android project ({})".format(self.name))
arch = self.ctx.archs[0]
python_install_dir = self.ctx.get_python_install_dir()
from_crystax = self.ctx.python_recipe.from_crystax
crystax_python_dir = join("crystax_python", "crystax_python")
if len(self.ctx.archs) > 1:
raise ValueError("LBRY/gradle support only one arch")
info("Copying LBRY/gradle build for {}".format(arch))
shprint(sh.rm, "-rf", self.dist_dir)
shprint(sh.cp, "-r", self.build_dir, self.dist_dir)
# either the build use environemnt variable (ANDROID_HOME)
# or the local.properties if exists
with current_directory(self.dist_dir):
with open('local.properties', 'w') as fileh:
fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
with current_directory(self.dist_dir):
info("Copying Python distribution")
if not exists("private") and not from_crystax:
ensure_dir("private")
if not exists("crystax_python") and from_crystax:
ensure_dir(crystax_python_dir)
hostpython = sh.Command(self.ctx.hostpython)
if not from_crystax:
try:
shprint(hostpython,
'-OO',
'-m',
'compileall',
python_install_dir,
_tail=10,
_filterout="^Listing")
except sh.ErrorReturnCode:
pass
if not exists('python-install'):
shprint(sh.cp, '-a', python_install_dir,
'./python-install')
self.distribute_libs(arch, [self.ctx.get_libs_dir(arch.arch)])
self.distribute_javaclasses(self.ctx.javaclass_dir,
dest_dir=join("src", "main", "java"))
if not from_crystax:
info("Filling private directory")
if not exists(join("private", "lib")):
info("private/lib does not exist, making")
shprint(sh.cp, "-a", join("python-install", "lib"),
"private")
shprint(sh.mkdir, "-p", join("private", "include",
"python2.7"))
libpymodules_fn = join("libs", arch.arch, "libpymodules.so")
if exists(libpymodules_fn):
shprint(sh.mv, libpymodules_fn, 'private/')
shprint(
sh.cp,
join('python-install', 'include',
'python2.7', 'pyconfig.h'),
join('private', 'include', 'python2.7/'))
info('Removing some unwanted files')
shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so'))
shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig'))
libdir = join(self.dist_dir, 'private', 'lib', 'python2.7')
site_packages_dir = join(libdir, 'site-packages')
with current_directory(libdir):
removes = []
for dirname, root, filenames in walk("."):
for filename in filenames:
for suffix in EXCLUDE_EXTS:
if filename.endswith(suffix):
removes.append(filename)
shprint(sh.rm, '-f', *removes)
info('Deleting some other stuff not used on android')
# To quote the original distribute.sh, 'well...'
shprint(sh.rm, '-rf', 'lib2to3')
shprint(sh.rm, '-rf', 'idlelib')
for filename in glob.glob('config/libpython*.a'):
shprint(sh.rm, '-f', filename)
shprint(sh.rm, '-rf', 'config/python.o')
else: # Python *is* loaded from crystax
ndk_dir = self.ctx.ndk_dir
py_recipe = self.ctx.python_recipe
python_dir = join(ndk_dir, 'sources', 'python',
py_recipe.version, 'libs', arch.arch)
shprint(sh.cp, '-r', join(python_dir, 'stdlib.zip'),
crystax_python_dir)
shprint(sh.cp, '-r', join(python_dir, 'modules'),
crystax_python_dir)
shprint(sh.cp, '-r', self.ctx.get_python_install_dir(),
join(crystax_python_dir, 'site-packages'))
info('Renaming .so files to reflect cross-compile')
site_packages_dir = join(crystax_python_dir, "site-packages")
find_ret = shprint(sh.find, site_packages_dir, '-iname',
'*.so')
filenames = find_ret.stdout.decode('utf-8').split('\n')[:-1]
for filename in filenames:
parts = filename.split('.')
if len(parts) <= 2:
continue
shprint(sh.mv, filename, filename.split('.')[0] + '.so')
site_packages_dir = join(abspath(curdir), site_packages_dir)
if 'sqlite3' not in self.ctx.recipe_build_order:
with open('blacklist.txt', 'a') as fileh:
fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
self.strip_libraries(arch)
self.fry_eggs(site_packages_dir)
super(LbryBootstrap, self).run_distribute()
def run_hook(kind, vmname, diskname):
if config['hooks'][kind] is not None:
sh.Command(config['hooks'][kind])(kind, vmname, diskname)
import sh
dpkgSig = sh.Command("dpkg-sig").bake(s="builder", _fg=True)
def apk(self, args):
"""Create an APK using the given distribution."""
ctx = self.ctx
dist = self._dist
# Manually fixing these arguments at the string stage is
# unsatisfactory and should probably be changed somehow, but
# we can't leave it until later as the build.py scripts assume
# they are in the current directory.
fix_args = ('--dir', '--private', '--add-jar', '--add-source',
'--whitelist', '--blacklist', '--presplash', '--icon')
unknown_args = args.unknown_args
for i, arg in enumerate(unknown_args):
argx = arg.split('=')
if argx[0] in fix_args:
if len(argx) > 1:
unknown_args[i] = '='.join(
(argx[0], realpath(expanduser(argx[1]))))
elif i + 1 < len(unknown_args):
unknown_args[i + 1] = realpath(
expanduser(unknown_args[i + 1]))
env = os.environ.copy()
if args.build_mode == 'release':
if args.keystore:
env['P4A_RELEASE_KEYSTORE'] = realpath(
expanduser(args.keystore))
if args.signkey:
env['P4A_RELEASE_KEYALIAS'] = args.signkey
if args.keystorepw:
env['P4A_RELEASE_KEYSTORE_PASSWD'] = args.keystorepw
if args.signkeypw:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.signkeypw
elif args.keystorepw and 'P4A_RELEASE_KEYALIAS_PASSWD' not in env:
env['P4A_RELEASE_KEYALIAS_PASSWD'] = args.keystorepw
build = imp.load_source('build', join(dist.dist_dir, 'build.py'))
with current_directory(dist.dist_dir):
self.hook("before_apk_build")
os.environ["ANDROID_API"] = str(self.ctx.android_api)
build_args = build.parse_args(args.unknown_args)
self.hook("after_apk_build")
self.hook("before_apk_assemble")
build_type = ctx.java_build_tool
if build_type == 'auto':
info('Selecting java build tool:')
build_tools_versions = os.listdir(
join(ctx.sdk_dir, 'build-tools'))
build_tools_versions = sorted(build_tools_versions,
key=LooseVersion)
build_tools_version = build_tools_versions[-1]
info(('Detected highest available build tools '
'version to be {}').format(build_tools_version))
if build_tools_version >= '25.0' and exists('gradlew'):
build_type = 'gradle'
info(' Building with gradle, as gradle executable is '
'present')
else:
build_type = 'ant'
if build_tools_version < '25.0':
info((' Building with ant, as the highest '
'build-tools-version is only {}'
).format(build_tools_version))
else:
info(' Building with ant, as no gradle executable '
'detected')
if build_type == 'gradle':
# gradle-based build
env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir
env["ANDROID_HOME"] = self.ctx.sdk_dir
gradlew = sh.Command('./gradlew')
if exists('/usr/bin/dos2unix'):
# .../dists/bdisttest_python3/gradlew
# .../build/bootstrap_builds/sdl2-python3crystax/gradlew
# if docker on windows, gradle contains CRLF
output = shprint(sh.Command('dos2unix'),
gradlew._path.decode('utf8'),
_tail=20,
_critical=True,
_env=env)
if args.build_mode == "debug":
gradle_task = "assembleDebug"
elif args.build_mode == "release":
gradle_task = "assembleRelease"
else:
raise BuildInterruptingException(
"Unknown build mode {} for apk()".format(
args.build_mode))
output = shprint(gradlew,
gradle_task,
_tail=20,
_critical=True,
_env=env)
# gradle output apks somewhere else
# and don't have version in file
apk_dir = join(dist.dist_dir, "build", "outputs", "apk",
args.build_mode)
apk_glob = "*-{}.apk"
apk_add_version = True
else:
# ant-based build
try:
ant = sh.Command('ant')
except sh.CommandNotFound:
raise BuildInterruptingException(
'Could not find ant binary, please install it '
'and make sure it is in your $PATH.')
output = shprint(ant,
args.build_mode,
_tail=20,
_critical=True,
_env=env)
apk_dir = join(dist.dist_dir, "bin")
apk_glob = "*-*-{}.apk"
apk_add_version = False
self.hook("after_apk_assemble")
info_main('# Copying APK to current directory')
apk_re = re.compile(r'.*Package: (.*\.apk)$')
apk_file = None
for line in reversed(output.splitlines()):
m = apk_re.match(line)
if m:
apk_file = m.groups()[0]
break
if not apk_file:
info_main('# APK filename not found in build output. Guessing...')
if args.build_mode == "release":
suffixes = ("release", "release-unsigned")
else:
suffixes = ("debug", )
for suffix in suffixes:
apks = glob.glob(join(apk_dir, apk_glob.format(suffix)))
if apks:
if len(apks) > 1:
info('More than one built APK found... guessing you '
'just built {}'.format(apks[-1]))
apk_file = apks[-1]
break
else:
raise BuildInterruptingException(
'Couldn\'t find the built APK')
info_main('# Found APK file: {}'.format(apk_file))
if apk_add_version:
info('# Add version number to APK')
apk_name, apk_suffix = basename(apk_file).split("-", 1)
apk_file_dest = "{}-{}-{}".format(apk_name, build_args.version,
apk_suffix)
info('# APK renamed to {}'.format(apk_file_dest))
shprint(sh.cp, apk_file, apk_file_dest)
else:
shprint(sh.cp, apk_file, './')
def do_python_build(self, arch):
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, self.ctx.hostpython, self.get_build_dir(arch.arch))
shprint(sh.cp, self.ctx.hostpgen, self.get_build_dir(arch.arch))
hostpython = join(self.get_build_dir(arch.arch), 'hostpython')
hostpgen = join(self.get_build_dir(arch.arch), 'hostpython')
with current_directory(self.get_build_dir(arch.arch)):
hostpython_recipe = Recipe.get_recipe('hostpython2', self.ctx)
shprint(sh.cp, join(hostpython_recipe.get_recipe_dir(), 'Setup'), 'Modules')
env = arch.get_env()
env['HOSTARCH'] = 'arm-eabi'
env['BUILDARCH'] = shprint(sh.gcc, '-dumpmachine').stdout.decode('utf-8').split('\n')[0]
env['CFLAGS'] = ' '.join([env['CFLAGS'], '-DNO_MALLINFO'])
# TODO need to add a should_build that checks if optional
# dependencies have changed (possibly in a generic way)
if 'openssl' in self.ctx.recipe_build_order:
r = Recipe.get_recipe('openssl', self.ctx)
openssl_build_dir = r.get_build_dir(arch.arch)
setuplocal = join('Modules', 'Setup.local')
shprint(sh.cp, join(self.get_recipe_dir(), 'Setup.local-ssl'), setuplocal)
shprint(sh.sed, '-i.backup', 's#^SSL=.*#SSL={}#'.format(openssl_build_dir), setuplocal)
env['OPENSSL_VERSION'] = r.version
if 'sqlite3' in self.ctx.recipe_build_order:
# Include sqlite3 in python2 build
r = Recipe.get_recipe('sqlite3', self.ctx)
i = ' -I' + r.get_build_dir(arch.arch)
l = ' -L' + r.get_lib_dir(arch) + ' -lsqlite3'
# Insert or append to env
f = 'CPPFLAGS'
env[f] = env[f] + i if f in env else i
f = 'LDFLAGS'
env[f] = env[f] + l if f in env else l
# NDK has langinfo.h but doesn't define nl_langinfo()
env['ac_cv_header_langinfo_h'] = 'no'
configure = sh.Command('./configure')
shprint(configure,
'--host={}'.format(env['HOSTARCH']),
'--build={}'.format(env['BUILDARCH']),
# 'OPT={}'.format(env['OFLAG']),
'--prefix={}'.format(realpath('./python-install')),
'--enable-shared',
'--disable-toolbox-glue',
'--disable-framework',
_env=env)
# tito left this comment in the original source. It's still true!
# FIXME, the first time, we got a error at:
# python$EXE ../../Tools/scripts/h2py.py -i '(u_long)' /usr/include/netinet/in.h
# /home/tito/code/python-for-android/build/python/Python-2.7.2/python: 1: Syntax error: word unexpected (expecting ")")
# because at this time, python is arm, not x86. even that, why /usr/include/netinet/in.h is used ?
# check if we can avoid this part.
make = sh.Command(env['MAKE'].split(' ')[0])
print('First install (expected to fail...')
try:
shprint(make, '-j5', 'install', 'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
except sh.ErrorReturnCode_2:
print('First python2 make failed. This is expected, trying again.')
print('Second install (expected to work)')
shprint(sh.touch, 'python.exe', 'python')
shprint(make, '-j5', 'install', 'HOSTPYTHON={}'.format(hostpython),
'HOSTPGEN={}'.format(hostpgen),
'CROSS_COMPILE_TARGET=yes',
'INSTSONAME=libpython2.7.so',
_env=env)
if is_darwin():
shprint(sh.cp, join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'Lib'))
shprint(sh.cp, join(self.get_recipe_dir(), 'patches', '_scproxy.py'),
join('python-install', 'lib', 'python2.7'))
# reduce python
for dir_name in ('test', join('json', 'tests'), 'lib-tk',
join('sqlite3', 'test'), join('unittest, test'),
join('lib2to3', 'tests'), join('bsddb', 'tests'),
join('distutils', 'tests'), join('email', 'test'),
'curses'):
shprint(sh.rm, '-rf', join('python-install',
'lib', 'python2.7', dir_name))
ALIGNMENT_TEMP_PDB = "aligned.pdb"
ALIGNMENT_OUTPUT = "alignment.txt"
BIOJAVA_RUNTIME_ARGS = [
"-Xmx500M",
"-cp",
BIOJAVA_CLASSPATH,
]
BIOJAVA_COMMON_ARGS = [
"-autoFetch", "false", "-pdbDirSplit", "false", "-printFatCat",
"-outputPDB", "-outFile", ALIGNMENT_TEMP_PDB
]
_java_cmd = sh.Command(JAVA_RUNTIME)
_biojava_base_cmd = _java_cmd.bake(*BIOJAVA_RUNTIME_ARGS)
def run_biojava_alignment(cmd_class,
code1,
code2,
alignment,
transformed,
superposed,
pdb1=None,
pdb2=None,
log=None):
try:
exec_dir = tempfile.mkdtemp()
from StringIO import StringIO
from PIL import Image
from pdfminer.pdfpage import PDFPage
import sh
from django.utils.translation import ugettext_lazy as _
from common.utils import fs_cleanup, mkstemp
from ..classes import ConverterBase
from ..exceptions import PageCountError
from ..settings import setting_pdftoppm_path
try:
pdftoppm = sh.Command(setting_pdftoppm_path.value)
except sh.CommandNotFound:
pdftoppm = None
else:
pdftoppm = pdftoppm.bake('-jpeg')
Image.init()
logger = logging.getLogger(__name__)
class IteratorIO(object):
def __init__(self, iterator):
self.file_buffer = StringIO()
for chunk in iterator:
self.file_buffer.write(chunk)
def install(self):
arch = self.filtered_archs[0]
build_dir = self.get_build_dir(arch.arch)
hostpython = sh.Command(self.ctx.hostpython)
with cd(build_dir):
shprint(hostpython, "setup.py", "install")
# standard library imports
import sys
from pathlib import Path
# third-party imports
import pytest
import sh
# module imports
from . import INGEST_OUTPUTS
from . import find_homology_files
from . import help_check
from . import print_docstring
# global constants
azulejo = sh.Command("azulejo")
SUBCOMMAND = "homology"
def test_subcommand_help():
"""Test subcommand help message."""
help_check(SUBCOMMAND)
@print_docstring()
def test_homology(datadir_mgr, capsys):
"""Test homology clustering, MSA, and tree building."""
with capsys.disabled():
with datadir_mgr.in_tmp_dir(
inpathlist=INGEST_OUTPUTS,
save_outputs=True,
def run_pymodules_install(ctx, modules):
modules = list(filter(ctx.not_has_package, modules))
if not modules:
info('There are no Python modules to install, skipping')
return
info('The requirements ({}) don\'t have recipes, attempting to install '
'them with pip'.format(', '.join(modules)))
info('If this fails, it may mean that the module has compiled '
'components and needs a recipe.')
venv = sh.Command(ctx.virtualenv)
with current_directory(join(ctx.build_dir)):
shprint(
venv, '--python=python{}'.format(
ctx.python_recipe.major_minor_version_string.partition(".")
[0]), 'venv')
info('Creating a requirements.txt file for the Python modules')
with open('requirements.txt', 'w') as fileh:
for module in modules:
key = 'VERSION_' + module
if key in environ:
line = '{}=={}\n'.format(module, environ[key])
else:
line = '{}\n'.format(module)
fileh.write(line)
base_env = copy.copy(os.environ)
base_env["PYTHONPATH"] = ctx.get_site_packages_dir()
info('Upgrade pip to latest version')
shprint(sh.bash,
'-c', ("source venv/bin/activate && pip install -U pip"),
_env=copy.copy(base_env))
info('Install Cython in case one of the modules needs it to build')
shprint(sh.bash,
'-c', ("venv/bin/pip install Cython"),
_env=copy.copy(base_env))
# Get environment variables for build (with CC/compiler set):
standard_recipe = CythonRecipe()
standard_recipe.ctx = ctx
# (note: following line enables explicit -lpython... linker options)
standard_recipe.call_hostpython_via_targetpython = False
recipe_env = standard_recipe.get_recipe_env(ctx.archs[0])
env = copy.copy(base_env)
env.update(recipe_env)
info('Installing Python modules with pip')
info('IF THIS FAILS, THE MODULES MAY NEED A RECIPE. '
'A reason for this is often modules compiling '
'native code that is unaware of Android cross-compilation '
'and does not work without additional '
'changes / workarounds.')
# Make sure our build package dir is available, and the virtualenv
# site packages come FIRST (for the proper pip version):
env["PYTHONPATH"] += ":" + ctx.get_site_packages_dir()
env["PYTHONPATH"] = os.path.abspath(
join(ctx.build_dir, "venv", "lib",
"python" + ctx.python_recipe.major_minor_version_string,
"site-packages")) + ":" + env["PYTHONPATH"]
shprint(sh.bash,
'-c',
("source venv/bin/activate && " +
"pip install -v --target '{0}' --no-deps -r requirements.txt"
).format(ctx.get_site_packages_dir().replace("'", "'\"'\"'")),
_env=copy.copy(env))
# Strip object files after potential Cython or native code builds:
standard_recipe.strip_object_files(ctx.archs[0],
env,
build_dir=ctx.build_dir)
def main():
logger = logging.getLogger('dobutton')
s = None
while not s:
for serial_dev in set(itertools.chain.from_iterable(comports())):
try:
# print("Trying {0}".format(serial_dev))
s = serial.Serial(serial_dev, baudrate=9600, timeout=0.25)
connected = False
tries = 0
while tries < 3:
if "IM A BUTTON" in s.readline():
# print("FOUND A BUTTON")
connected = True
while True:
write(s, SUCCESS)
time.sleep(.1)
if s.readline().strip() == "SUCCESS":
print("DoButton found!")
break
break
tries += 1
else:
s.close()
continue
if connected:
break
except serial.serialutil.SerialException as exc:
# Couldn't connect, skip it.
# print("Couldn't connect to", serial_dev)
# print(exc.message)
continue
else:
# Tried all the devices, so sleep a bit before trying again.
time.sleep(1)
print(".", end='')
sys.stdout.flush()
# Get any button messages:
while True:
try:
line = s.readline()
except (OSError, serial.serialutil.SerialException, ValueError) as exc:
# print(exc.message)
# Arduino got unplugged, so start over.
return main()
if "PRESSED" in line:
line = None
# Send a message to the button that it's working.
write(s, WORKING)
# Call whatever script(s) are in ~/.dobutton/.
for command in get_executable():
try:
# Note: Don't background these, so they run in sequence
# and the "WORKING" message means something
print("Running", command)
run = sh.Command(command)
run()
print("Finished successfully.")
except sh.ErrorReturnCode as exc:
# If any error in the executables...
# Log it
print(exc.message)
logger.error("ERROR: {0} exited with a nonzero exit status.".format(command))
logger.error(exc.stdout)
logger.error(exc.stderr)
# And inform the button.
write(s, ERROR)
# And give up running more commands.
break
else:
# When they're all done, and they all succeed, send the success message.
write(s, SUCCESS)
def run(self,
i,
restart_file=None,
use_restart=True,
multi_node=False,
num_cores=8,
overwrite_data=False,
save_run=False,
run_idb=False,
nice_score=0,
mpirun_opts=''):
"""Run the model.0
`num_cores`: Number of mpi cores to distribute over.
`restart_file` (optional): A path to a valid restart archive. If None and `use_restart=True`,
restart file (i-1) will be used.
`save_run`: If True, copy the entire working directory over to GFDL_DATA
so that the run can rerun without the python script.
(This uses a lot of data storage!)
"""
self.clear_rundir()
indir = P(self.rundir, 'INPUT')
outdir = P(self.datadir, self.runfmt % i)
resdir = P(self.rundir, 'RESTART')
if self.check_for_existing_output(i):
if overwrite_data:
self.log.warning(
'Data for run %d already exists and overwrite_data is True. Overwriting.'
% i)
sh.rm('-r', outdir)
else:
self.log.warn(
'Data for run %d already exists but overwrite_data is False. Stopping.'
% i)
return False
# make the output run folder and copy over the input files
mkdir([indir, resdir, self.restartdir])
self.codebase.write_source_control_status(
P(self.rundir, 'git_hash_used.txt'))
self.write_namelist(self.rundir)
self.write_field_table(self.rundir)
self.write_diag_table(self.rundir)
for filename in self.inputfiles:
sh.cp([filename, P(indir, os.path.split(filename)[1])])
if multi_node:
mpirun_opts += ' -bootstrap pbsdsh -f $PBS_NODEFILE'
if use_restart and not restart_file and i == 1:
# no restart file specified, but we are at first run number
self.log.warn(
'use_restart=True, but restart_file not specified. As this is run 1, assuming spin-up from namelist stated initial conditions so continuing.'
)
use_restart = False
if use_restart:
if not restart_file:
# get the restart from previous iteration
restart_file = self.get_restart_file(i - 1)
if not os.path.isfile(restart_file):
self.log.error('Restart file not found, expecting file %r' %
restart_file)
raise IOError('Restart file not found, expecting file %r' %
restart_file)
else:
self.log.info('Using restart file %r' % restart_file)
self.extract_restart_archive(restart_file, indir)
else:
self.log.info('Running without restart file')
restart_file = None
vars = {
'rundir': self.rundir,
'execdir': self.codebase.builddir,
'executable': self.codebase.executable_name,
'env_source': self.env_source,
'mpirun_opts': mpirun_opts,
'num_cores': num_cores,
'run_idb': run_idb,
'nice_score': nice_score
}
runscript = self.templates.get_template('run.sh')
# employ the template to create a runscript
t = runscript.stream(**vars).dump(P(self.rundir, 'run.sh'))
def _outhandler(line):
handled = self.emit('run:output', self, line)
if not handled: # only log the output when no event handler is used
self.log_output(line)
self.emit('run:ready', self, i)
self.log.info("Beginning run %d" % i)
try:
#for line in sh.bash(P(self.rundir, 'run.sh'), _iter=True, _err_to_out=True):
proc = sh.bash(P(self.rundir, 'run.sh'),
_bg=True,
_out=_outhandler,
_err_to_out=True)
self.log.info('process running as {}'.format(proc.process.pid))
proc.wait()
completed = True
except KeyboardInterrupt as e:
self.log.error("Manual interrupt, killing process.")
proc.process.terminate()
proc.wait()
#log.info("Cleaning run directory.")
#self.clear_rundir()
raise e
except sh.ErrorReturnCode as e:
completed = False
self.log.error("Run %d failed. See log for details." % i)
self.log.error("Error: %r" % e)
self.emit('run:failed', self)
raise FailedRunError()
self.emit('run:complete', self, i)
self.log.info('Run %d complete' % i)
mkdir(outdir)
if num_cores > 1:
# use postprocessing tool to combine the output from several cores
codebase_combine_script = P(self.codebase.builddir,
'mppnccombine_run.sh')
if not os.path.exists(codebase_combine_script):
self.log.warning(
'combine script does not exist in the commit you are running Isca from. Falling back to using $GFDL_BASE mppnccombine_run.sh script'
)
sh.ln('-s',
P(GFDL_BASE, 'postprocessing', 'mppnccombine_run.sh'),
codebase_combine_script)
combinetool = sh.Command(codebase_combine_script)
for file in self.diag_table.files:
netcdf_file = '%s.nc' % file
filebase = P(self.rundir, netcdf_file)
combinetool(self.codebase.builddir, filebase)
# copy the combined netcdf file into the data archive directory
sh.cp(filebase, P(outdir, netcdf_file))
# remove all netcdf fragments from the run directory
sh.rm(glob.glob(filebase + '*'))
self.log.debug('%s combined and copied to data directory' %
netcdf_file)
for restart in glob.glob(P(resdir, '*.res.nc.0000')):
restartfile = restart.replace('.0000', '')
combinetool(self.codebase.builddir, restartfile)
sh.rm(glob.glob(restartfile + '.????'))
self.log.debug("Restart file %s combined" % restartfile)
self.emit('run:combined', self, i)
else:
for file in self.diag_table.files:
netcdf_file = '%s.nc' % file
filebase = P(self.rundir, netcdf_file)
sh.cp(filebase, P(outdir, netcdf_file))
sh.rm(glob.glob(filebase + '*'))
self.log.debug('%s copied to data directory' % netcdf_file)
# make the restart archive and delete the restart files
self.make_restart_archive(self.get_restart_file(i), resdir)
sh.rm('-r', resdir)
if save_run:
# copy the complete run directory to GFDL_DATA so that the run can
# be recreated without the python script if required
mkdir(resdir)
sh.cp(['-a', self.rundir, outdir])
else:
# just save some useful diagnostic information
self.write_namelist(outdir)
self.write_field_table(outdir)
self.write_diag_table(outdir)
self.codebase.write_source_control_status(
P(outdir, 'git_hash_used.txt'))
self.clear_rundir()
self.emit('run:finished', self, i)
return True
def regenerate_files(generate_tests_bin, test_config):
print("Regenerating files...")
sh.Command(generate_tests_bin)(
yaml_config_path=test_config,
_fg=True)
