def install(robustus, requirement_specifier, rob_file, ignore_index):
# check if already installed
pocketsphinx = os.path.join(robustus.env, 'lib/python2.7/site-packages/pocketsphinx.so')
if os.path.isfile(pocketsphinx):
return
cwd = os.getcwd()
archive = None
try:
os.chdir(robustus.cache)
build_dir = os.path.join(robustus.cache, 'pocketsphinx-%s' % requirement_specifier.version)
if not os.path.isfile(os.path.join(build_dir, 'configure')):
archive = robustus.download('pocketsphinx', requirement_specifier.version)
unpack(archive)
# unfortunately we can't cache pocketsphinx, it has to be rebuild after reconfigure
logging.info('Building pocketsphinx')
os.chdir(build_dir)
sphinxbase_dir = os.path.join(robustus.cache, 'sphinxbase-%s/' % requirement_specifier.version)
retcode = run_shell('./configure'
+ (' --prefix=%s' % robustus.env)
+ (' --with-python=%s' % os.path.join(robustus.env, 'bin/python'))
+ (' --with-sphinxbase=%s' % sphinxbase_dir)
+ (' --with-sphinxbase-build=%s' % sphinxbase_dir),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('pocketsphinx configure failed')
retcode = run_shell('make clean && make', shell=True, verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('pocketsphinx build failed')
logging.info('Installing pocketsphinx into virtualenv')
retcode = run_shell('make install', shell=True, verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('pocketsphinx install failed')
fix_rpath(robustus, robustus.env, pocketsphinx, os.path.join(robustus.env, 'lib'))
# there is a super weird bug, first import of pocketsphinx fails http://sourceforge.net/p/cmusphinx/bugs/284/
write_file(os.path.join(robustus.env, 'lib/python2.7/site-packages/wrap_pocketsphinx.py'),
'w',
'try:\n'
+ ' from pocketsphinx import *\n'
+ 'except:\n'
+ ' pass\n'
+ 'from pocketsphinx import *\n')
except RequirementException:
safe_remove(build_dir)
finally:
if archive is not None:
safe_remove(archive)
os.chdir(cwd)
def prepare_dataset(url, md5sum, target_dir, annotation_path):
"""Download, unpack and create manifest file."""
data_dir = os.path.join(target_dir, 'data_thchs30')
if not os.path.exists(data_dir):
filepath = download(url, md5sum, target_dir)
unpack(filepath, target_dir)
os.remove(filepath)
else:
print(
"Skip downloading and unpacking. THCHS-30 data already exists in %s."
% target_dir)
create_annotation_text(data_dir, annotation_path)
def prepare_dataset(url, md5sum, target_dir, annotation_path):
"""Download, unpack and create manifest file."""
data_dir = os.path.join(target_dir, 'ST-CMDS-20170001_1-OS')
if not os.path.exists(data_dir):
filepath = download(url, md5sum, target_dir)
unpack(filepath, target_dir)
os.remove(filepath)
else:
print(
"Skip downloading and unpacking. Free ST-Chinese-Mandarin-Corpus data already exists in %s."
% target_dir)
create_annotation_text(data_dir, annotation_path)
def install(robustus, requirement_specifier, rob_file, ignore_index):
# If we are not on Travis, we only link to system-wide PySide (for now), and return
# The reason we do this hack for now is to avoid installing qt sdk on bstem
# So, on bstem and linux machines we will need to sudo apt-get install python-pyside
# On Mac machines: to be updated soon!
if 'TRAVIS' not in os.environ:
# Create links to system-wide PySide
if sys.platform.startswith('darwin'):
if os.path.isfile('/usr/local/lib/python2.7/site-packages/PySide/QtCore.so'):
logging.info('Linking pyside on macos')
ln('/usr/local/lib/python2.7/site-packages/sip.so',
os.path.join(robustus.env, 'lib/python2.7/site-packages/sip.so'), force = True)
ln('/usr/local/lib/python2.7/site-packages/PySide',
os.path.join(robustus.env, 'lib/python2.7/site-packages/PySide'), force = True)
else:
raise RequirementException('System-wide PySide is missing, run: brew install pyside')
else:
candidates = ['/usr/lib64/python2.7/site-packages/',
'/usr/lib/python2.7/dist-packages',
'/usr/local/lib/python2.7/dist-packages']
for c in candidates:
if os.path.isfile(os.path.join(c, 'PySide/QtCore.so')):
logging.info('Linking pyside in %s' % c)
ln(os.path.join(c, 'sip.so'),
os.path.join(robustus.env, 'lib/python2.7/site-packages/sip.so'), force = True)
ln(os.path.join(c, 'PySide'),
os.path.join(robustus.env, 'lib/python2.7/site-packages/PySide'), force = True)
break
else:
raise RequirementException('System-wide PySide is missing, run: sudo apt-get install python-pyside')
else:
# If we are on Travis, then:
# Install through wheeling
robustus.install_through_wheeling(requirement_specifier, rob_file, ignore_index)
# Need links to shared libraries
pyside_setup_dir = os.path.join(robustus.cache, 'pyside-setup-master')
if not os.path.isdir(pyside_setup_dir) and not ignore_index:
os.chdir(robustus.cache)
pyside_setup_archive = robustus.download('pyside-setup', 'master')
unpack(pyside_setup_archive)
cwd = os.getcwd()
try:
# run postinstall
if not os.path.isdir(pyside_setup_dir):
raise RequirementException('can\'t find pyside-%s in robustus cache' % requirement_specifier.version)
os.chdir(pyside_setup_dir)
retcode = subprocess.call([robustus.python_executable, 'pyside_postinstall.py', '-install'])
if retcode != 0:
raise RequirementException('failed to execute pyside postinstall script')
finally:
os.chdir(cwd)
def install(robustus, requirement_specifier, rob_file, ignore_index):
# check if already installed
sphinxbase = os.path.join(robustus.env, 'lib/python2.7/site-packages/sphinxbase.so')
if os.path.isfile(sphinxbase):
return
cwd = os.getcwd()
archive = None
try:
# build in cache
os.chdir(robustus.cache)
build_dir = os.path.join(robustus.cache, 'sphinxbase-%s' % requirement_specifier.version)
if not os.path.isfile(os.path.join(build_dir, 'configure')):
archive = robustus.download('sphinxbase', requirement_specifier.version)
unpack(archive)
# unfortunately we can't cache sphinxbase, it has to be rebuild after reconfigure
logging.info('Building sphinxbase')
os.chdir(build_dir)
# link python-config from system-wide installation, sphinxbase configure requires it
python_config = os.path.join(robustus.env, 'bin/python-config')
if not os.path.isfile(python_config):
ln('/usr/bin/python-config', python_config)
retcode = run_shell('./configure'
+ (' --prefix=%s' % robustus.env)
+ (' --with-python=%s' % os.path.join(robustus.env, 'bin/python')),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('sphinxbase configure failed')
retcode = run_shell('make clean && make', shell=True, verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('sphinxbase build failed')
logging.info('Installing sphinxbase into virtualenv')
retcode = run_shell('make install', shell=True, verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('sphinxbase install failed')
fix_rpath(robustus, robustus.env, sphinxbase, os.path.join(robustus.env, 'lib'))
except RequirementException:
safe_remove(build_dir)
finally:
if archive is not None:
safe_remove(archive)
os.chdir(cwd)
def read_class_data(path, label=None):
'''
Label may come from the data itself, may be assigned at run time
'''
if os.path.exists(path):
if os.path.isdir(path):
paths = [os.path.join(path, f) for f in os.listdir(path)]
else:
paths = [path]
else:
print 'Given path does not exist.'
return
doc = doc_file()
stemmer = PorterStemmer()
instances = []
for p in paths:
doc.path = p
for raw_record in doc:
record = unpack(raw_record, ',')
text = record[3].strip('"')
inst = {'tokens': [], 'label': ''}
for t in wordpunct_tokenize(text):
stem_t = stemmer.stem(t.lower())
if stem_t[0].islower():
inst['tokens'].append(stem_t)
else:
continue
inst['label'] = label
instances.append(inst)
return instances
def install(robustus, requirement_specifier, rob_file, ignore_index):
patchelf_cache_dir = os.path.abspath(os.path.join(robustus.cache, 'patchelf-%s' % requirement_specifier.version))
def in_cache():
return os.path.isfile(os.path.join(patchelf_cache_dir, 'patchelf'))
cwd = os.getcwd()
if not in_cache() and not ignore_index:
logging.info('Downloading patchelf')
patchelf_tgz = robustus.download('patchelf', requirement_specifier.version)
patchelf_archive_name = unpack(patchelf_tgz)
logging.info('Building patchelf')
os.chdir(patchelf_archive_name)
run_shell(['./bootstrap.sh'], verbose=robustus.settings['verbosity'] >= 1)
run_shell(['./configure'], verbose=robustus.settings['verbosity'] >= 1)
run_shell(['make'], verbose=robustus.settings['verbosity'] >= 1)
if os.path.isdir(patchelf_cache_dir):
shutil.rmtree(patchelf_cache_dir)
os.mkdir(patchelf_cache_dir)
cp('./src/patchelf', patchelf_cache_dir)
os.chdir(os.path.pardir)
os.remove(patchelf_tgz)
shutil.rmtree(patchelf_archive_name)
os.chdir(cwd)
if in_cache():
patchelf_install_dir = os.path.join(robustus.env, 'bin')
cp(os.path.join(patchelf_cache_dir, 'patchelf'), patchelf_install_dir)
else:
raise RequirementException('can\'t find patchelf-%s in robustus cache' % requirement_specifier.version)
def prepare_dataset(url, md5sum, target_dir, annotation_path):
"""Download, unpack and create manifest file."""
data_dir = os.path.join(target_dir, 'data_aishell')
if not os.path.exists(data_dir):
filepath = download(url, md5sum, target_dir)
unpack(filepath, target_dir)
# unpack all audio tar files
audio_dir = os.path.join(data_dir, 'wav')
for subfolder, _, filelist in sorted(os.walk(audio_dir)):
for ftar in filelist:
unpack(os.path.join(subfolder, ftar), subfolder, True)
os.remove(filepath)
else:
print(
"Skip downloading and unpacking. Aishell data already exists in %s."
% target_dir)
create_annotation_text(data_dir, annotation_path)
def install(robustus, requirement_specifier, rob_file, ignore_index):
bullet_cache_dir = os.path.abspath(
os.path.join(robustus.cache,
'bullet-%s' % requirement_specifier.version))
def in_cache():
return os.path.isfile(
os.path.join(bullet_cache_dir, 'lib/libBulletCollision.a'))
if not in_cache() and not ignore_index:
cwd = os.getcwd()
bullet_archive = None
bullet_archive_name = None
try:
bullet_archive = robustus.download('bullet',
requirement_specifier.version)
bullet_archive_name = unpack(bullet_archive)
logging.info('Building bullet')
os.chdir(bullet_archive_name)
run_shell([
'cmake', '.', '-G', "Unix Makefiles",
'-DCMAKE_INSTALL_PREFIX=%s' % bullet_cache_dir,
'-DCMAKE_CXX_FLAGS=-fPIC',
'-DBUILD_NVIDIA_OPENCL_DEMOS:BOOL=OFF',
'-DBUILD_INTEL_OPENCL_DEMOS:BOOL=OFF',
'-DCMAKE_C_COMPILER=gcc', '-DCMAKE_CXX_COMPILER=g++'
],
verbose=robustus.settings['verbosity'] >= 1)
retcode = run_shell(['make', '-j4'],
shell=False,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('bullet build failed')
retcode = run_shell(['make', 'install'],
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('bullet "make install" failed')
finally:
safe_remove(bullet_archive)
safe_remove(bullet_archive_name)
os.chdir(cwd)
if in_cache():
# install bullet somewhere into venv
bullet_install_dir = os.path.join(
robustus.env, 'lib/bullet-%s' % requirement_specifier.version)
if os.path.exists(bullet_install_dir):
shutil.rmtree(bullet_install_dir)
shutil.copytree(bullet_cache_dir, bullet_install_dir)
else:
raise RequirementException('can\'t find bullet-%s in robustus cache' %
requirement_specifier.version)
def get_one_product_review(src, des, pid):
for line in iter(src.readline, ''):
items = unpack(line, '\t')
try:
product_id = items[1]
title = items[-2]
review = items[-1]
except IndexError:
continue
if product_id == pid:
des.write(title + '\n' + review + '\n')
return
def get_n_each_sample(src, des, n):
c = [0, 0, 0, 0, 0]
while c[0] < n or c[1] < n or c[2] < n or c[3] < n or c[4] < n:
line = src.readline()
try:
rate = int(math.ceil(float(unpack(line, '\t')[5])))
index = rate - 1
if c[index] < n:
des.write(line)
c[index] = c[index] + 1
except IndexError:
continue
return
def set_attribute_type(src, des, a_name, a_type):
for line in iter(src.readline, ''):
if line[:10] != '@attribute':
des.write(line)
continue
items = unpack(line.strip('\n'), ' ')
try:
if items[-2] == a_name:
items[-1] = a_type
des.write(' '.join(items) + '\n')
else:
des.write(line)
except IndexError:
des.write(line)
return
def install(robustus, requirement_specifier, rob_file, ignore_index):
bullet_cache_dir = os.path.abspath(os.path.join(robustus.cache, 'bullet-%s' % requirement_specifier.version))
def in_cache():
return os.path.isfile(os.path.join(bullet_cache_dir, 'lib/libBulletCollision.a'))
if not in_cache() and not ignore_index:
cwd = os.getcwd()
bullet_archive = None
bullet_archive_name = None
try:
bullet_archive = robustus.download('bullet', requirement_specifier.version)
bullet_archive_name = unpack(bullet_archive)
logging.info('Building bullet')
os.chdir(bullet_archive_name)
run_shell(['cmake', '.',
'-G', "Unix Makefiles",
'-DCMAKE_INSTALL_PREFIX=%s' % bullet_cache_dir,
'-DCMAKE_CXX_FLAGS=-fPIC',
'-DBUILD_NVIDIA_OPENCL_DEMOS:BOOL=OFF',
'-DBUILD_INTEL_OPENCL_DEMOS:BOOL=OFF',
'-DCMAKE_C_COMPILER=gcc',
'-DCMAKE_CXX_COMPILER=g++'],
verbose=robustus.settings['verbosity'] >= 1)
retcode = run_shell(['make', '-j4'],
shell=False,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('bullet build failed')
retcode = run_shell(['make', 'install'],
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('bullet "make install" failed')
finally:
safe_remove(bullet_archive)
safe_remove(bullet_archive_name)
os.chdir(cwd)
if in_cache():
# install bullet somewhere into venv
bullet_install_dir = os.path.join(robustus.env, 'lib/bullet-%s' % requirement_specifier.version)
if os.path.exists(bullet_install_dir):
shutil.rmtree(bullet_install_dir)
shutil.copytree(bullet_cache_dir, bullet_install_dir)
else:
raise RequirementException('can\'t find bullet-%s in robustus cache' % requirement_specifier.version)
def set_attribute_value(src, des, a_index, new_value, old_value):
for line in iter(src.readline, ''):
if line[0] in '@\n':
des.write(line)
continue
items = unpack(line.strip('\n'), ',')
if not old_value:
try:
items[a_index] = new_value
des.write(','.join(items) + '\n')
except IndexError:
des.write(line)
else:
try:
i = old_value.index(items[a_index])
items[a_index] = new_value[i]
des.write(','.join(items) + '\n')
except (IndexError, ValueError):
des.write(line)
return
def install(robustus, requirement_specifier, rob_file, ignore_index):
patchelf_cache_dir = os.path.abspath(
os.path.join(robustus.cache,
'patchelf-%s' % requirement_specifier.version))
def in_cache():
return os.path.isfile(os.path.join(patchelf_cache_dir, 'patchelf'))
cwd = os.getcwd()
if not in_cache() and not ignore_index:
logging.info('Downloading patchelf')
patchelf_tgz = robustus.download('patchelf',
requirement_specifier.version)
patchelf_archive_name = unpack(patchelf_tgz)
logging.info('Building patchelf')
os.chdir(patchelf_archive_name)
run_shell(['./bootstrap.sh'],
verbose=robustus.settings['verbosity'] >= 1)
run_shell(['./configure'], verbose=robustus.settings['verbosity'] >= 1)
run_shell(['make'], verbose=robustus.settings['verbosity'] >= 1)
if os.path.isdir(patchelf_cache_dir):
shutil.rmtree(patchelf_cache_dir)
os.mkdir(patchelf_cache_dir)
cp('./src/patchelf', patchelf_cache_dir)
os.chdir(os.path.pardir)
os.remove(patchelf_tgz)
shutil.rmtree(patchelf_archive_name)
os.chdir(cwd)
if in_cache():
patchelf_install_dir = os.path.join(robustus.env, 'bin')
cp(os.path.join(patchelf_cache_dir, 'patchelf'), patchelf_install_dir)
else:
raise RequirementException(
'can\'t find patchelf-%s in robustus cache' %
requirement_specifier.version)
def install(robustus, requirement_specifier, rob_file, ignore_index):
cwd = os.getcwd()
os.chdir(robustus.cache)
install_dir = os.path.join(robustus.cache, 'protobuf-%s' % requirement_specifier.version)
# try to download precompiled protobuf from the remote cache first
if not os.path.isdir(install_dir) and not ignore_index:
protobuf_archive = robustus.download_compiled_archive('protobuf', requirement_specifier.version)
if protobuf_archive is not None:
unpack(protobuf_archive)
logging.info('Initializing compiled protobuf')
# install into wheelhouse
if not os.path.exists(install_dir):
raise RequirementException("Failed to unpack precompiled protobuf archive")
if not os.path.isdir(install_dir) and not ignore_index:
archive_name = 'protobuf-%s.tar.gz' % requirement_specifier.version
if os.path.exists(archive_name):
safe_remove(archive_name)
# move sources to a folder in order to use a clean name for installation
src_dir = 'protobuf-%s' % requirement_specifier.version
if os.path.exists(src_dir):
safe_remove(src_dir)
run_shell(['wget', 'https://protobuf.googlecode.com/svn/rc/%s' % (archive_name,)],
verbose=robustus.settings['verbosity'] >= 1)
run_shell(['tar', 'zxvf', archive_name],
verbose=robustus.settings['verbosity'] >= 1)
if os.path.exists(src_dir+'_src'):
safe_remove(src_dir+'_src')
shutil.move(src_dir, src_dir+'_src')
src_dir += '_src'
os.chdir(src_dir)
if os.path.exists(install_dir):
safe_remove(install_dir)
os.mkdir(install_dir)
retcode = run_shell(['./configure', '--disable-shared',
'CFLAGS=-fPIC',
'CXXFLAGS=-fPIC',
'--prefix', install_dir],
verbose=robustus.settings['verbosity'] >= 1)
if retcode:
raise RequirementException('Failed to configure protobuf compilation')
retcode = run_shell('make', shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode:
raise RequirementException('Failed compile protobuf')
retcode = run_shell('make install', shell=True)
if retcode:
raise RequirementException('Failed install protobuf')
os.chdir(robustus.cache)
shutil.rmtree(src_dir)
venv_install_folder = os.path.join(robustus.env, 'protobuf')
if os.path.exists(venv_install_folder):
safe_remove(venv_install_folder)
shutil.copytree(install_dir, venv_install_folder)
executable_path = os.path.join(install_dir, 'bin', 'protoc')
ln(executable_path, os.path.join(robustus.env, 'bin', 'protoc'), force=True)
os.chdir(cwd)
# now install python part
robustus.install_through_wheeling(requirement_specifier, rob_file, ignore_index)
def install(robustus, requirement_specifier, rob_file, ignore_index):
assert requirement_specifier.name == 'ros_overlay'
packages = requirement_specifier.version.split(',')
cwd = os.getcwd()
try:
env_source = os.path.join(robustus.env, 'bin/activate')
_ros_dep_init(env_source, robustus)
# NOTE: If ROS is not installed, the following returns an empty string.
def get_ros_install_dir(env_source):
ret_code, output = run_shell(
'. "%s" && python -c "import ros ; print ros.__file__"' %
env_source,
shell=True,
return_output=True)
if ret_code != 0:
logging.info(
'get_ros_install_dir() failed: ret_code is %d: %s' %
(ret_code, output))
return ''
if len(output.splitlines()) != 1:
logging.info(
'get_ros_install_dir() failed: Too many lines in output: %s'
% output)
return ''
output_dirname = os.path.dirname(output)
ros_install_dir = os.path.abspath(
os.path.join(output_dirname, os.pardir, os.pardir, os.pardir,
os.pardir))
if not os.path.isdir(ros_install_dir):
logging.info(
'get_ros_install_dir() failed: ros_install_dir not a directory: %s'
% ros_install_dir)
return ''
return ros_install_dir
ros_install_dir = get_ros_install_dir(env_source)
req_name = "ros-installed-overlay"
ver_hash = requirement_specifier.version_hash()
logging.info(
'Hashing ROS overlay on (robustus.env, ver_hash, ros_install_dir) = ("%s", "%s", "%s")'
% (robustus.env, ver_hash, ros_install_dir))
req_hash = ros_utils.hash_path(robustus.env, ver_hash, ros_install_dir)
overlay_install_folder = os.path.join(robustus.cache,
'%s-%s' % (req_name, req_hash))
if not os.path.isdir(overlay_install_folder):
overlay_archive = robustus.download_compiled_archive(
req_name, req_hash)
if overlay_archive:
overlay_archive_name = unpack(overlay_archive)
logging.info('Initializing compiled ROS overlay')
# install into wheelhouse
safe_move(overlay_archive_name, overlay_install_folder)
safe_remove(overlay_archive)
else:
overlay_src_folder = _make_overlay_folder(robustus, req_hash)
os.chdir(overlay_src_folder)
logging.info('Building ros overlay in %s with versions %s'
' install folder %s' %
(overlay_src_folder, str(packages),
overlay_install_folder))
os.mkdir(os.path.join(overlay_src_folder, 'src'))
_get_sources(packages)
_ros_dep(env_source, robustus)
opencv_cmake_dir = _opencv_cmake_path(robustus)
ret_code = run_shell(
'. "%s" && export OpenCV_DIR="%s" && catkin_make_isolated'
' --install-space %s --install' %
(env_source, opencv_cmake_dir, overlay_install_folder) +
' --force-cmake --cmake-args -DCATKIN_ENABLE_TESTING=1 ',
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if ret_code != 0:
raise RequirementException('Error during catkin_make')
else:
logging.info('ROS overlay cached %s' % overlay_install_folder)
add_source_ref(robustus,
os.path.join(overlay_install_folder, 'setup.sh'))
except RequirementException:
if robustus.settings['debug']:
logging.info('Not removing folder %s due to debug flag.' %
overlay_src_folder)
else:
shutil.rmtree(overlay_src_folder, ignore_errors=True)
shutil.rmtree(overlay_install_folder, ignore_errors=True)
raise
finally:
os.chdir(cwd)
def install(robustus, requirement_specifier, rob_file, ignore_index):
ver, dist = requirement_specifier.version.split('.')
if platform.machine() == 'armv7l':
# specific code to link ros on bstem
ros_install_dir = os.path.join('/opt/bstem/bstem.ros', 'ros-install-%s' % requirement_specifier.version)
if os.path.isdir(ros_install_dir):
# check distro
if (ver == 'hydro' and dist == 'ros_comm'):
# Add ROS settings to activate file
logging.info('Using ROS system install %s' % ros_install_dir)
add_source_ref(robustus, os.path.join(ros_install_dir, 'setup.sh'))
return
else:
logging.warn('armv7l only uses hydro.ros_comm as a ROS system install.\n')
else:
logging.warn('No suitable ROS system install found.\n')
# check distro
if ver != 'hydro':
logging.warn('Robustus is only tested to install ROS hydro.\n'
'Still, it will try to install required distribution "%s"' % requirement_specifier.version)
# install dependencies, may throw
robustus.execute(['install',
'catkin_pkg==0.2.2',
'rosinstall==0.6.30',
'rosinstall_generator==0.1.4',
'wstool==0.0.4',
'empy==3.3.2',
'rosdep==0.10.27',
'sip'])
ros_src_dir = os.path.join(robustus.env, 'ros-src-%s' % requirement_specifier.version)
req_name = 'ros-install-%s' % requirement_specifier.version
req_hash = ros_utils.hash_path(robustus.env)
ros_install_dir = os.path.join(robustus.cache, '%s-%s' % (req_name, req_hash))
def in_cache():
return os.path.isdir(ros_install_dir)
rosdep = _install_ros_deps(robustus)
try:
cwd = os.getcwd()
logging.info('ROS install dir %s' % ros_install_dir)
# download and install compiled non-system ROS or, if necessary, build ROS
if not in_cache() and not ignore_index:
ros_archive = robustus.download_compiled_archive(req_name, req_hash)
if ros_archive:
ros_archive_name = unpack(ros_archive)
logging.info('Initializing compiled ROS in Robustus wheelhouse')
# install into wheelhouse
safe_move(ros_archive_name, ros_install_dir)
safe_remove(ros_archive)
else:
logging.info('Building ROS in Robustus wheelhouse')
if not os.path.isdir(ros_src_dir):
os.makedirs(ros_src_dir)
os.chdir(ros_src_dir)
rosinstall_generator = os.path.join(robustus.env, 'bin/rosinstall_generator')
retcode = run_shell(rosinstall_generator + ' %s --rosdistro %s' % (dist, ver)
+ ' --deps --wet-only > %s-%s-wet.rosinstall' % (dist, ver),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('Failed to generate rosinstall file')
wstool = os.path.join(robustus.env, 'bin/wstool')
retcode = run_shell(wstool + ' init -j2 src %s-%s-wet.rosinstall' % (dist, ver),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('Failed to build ROS')
# resolve dependencies
if sys.platform.startswith('darwin'):
ros_os_param = ''
else:
ros_os_param = '--os=ubuntu:%s' % _get_distribution()
retcode = run_shell(rosdep + ' install -r --from-paths src --ignore-src --rosdistro %s -y %s' %
(ver, ros_os_param),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
if platform.machine() == 'armv7l':
# Due to the lack of LISP machine for ARM we expect some failures
logging.info("No LISP on ARM. Expected not all dependencies to be installed.")
else:
raise RequirementException('Failed to resolve ROS dependencies')
# create catkin workspace
py_activate_file = os.path.join(robustus.env, 'bin', 'activate')
catkin_make_isolated = os.path.join(ros_src_dir, 'src/catkin/bin/catkin_make_isolated')
retcode = run_shell('. ' + py_activate_file + ' && ' +
catkin_make_isolated +
' --install-space %s --install' % ros_install_dir,
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('Failed to create catkin workspace for ROS')
logging.info('Removing ROS source/build directory %s' % ros_src_dir)
os.chdir(ros_install_dir) # NOTE: If this directory is not accessible, something is wrong.
shutil.rmtree(ros_src_dir, ignore_errors=False)
else:
logging.info('Using ROS from cache %s' % ros_install_dir)
os.chdir(cwd)
# Add ROS settings to activate file
add_source_ref(robustus, os.path.join(ros_install_dir, 'setup.sh'))
except RequirementException:
os.chdir(cwd)
if robustus.settings['debug']:
logging.info('Not removing folder %s due to debug flag.' % ros_src_dir)
logging.info('Not removing folder %s due to debug flag.' % ros_install_dir)
else:
shutil.rmtree(ros_src_dir, ignore_errors=True)
shutil.rmtree(ros_install_dir, ignore_errors=True)
raise
def install(robustus, requirement_specifier, rob_file, ignore_index):
if requirement_specifier.version != '1.8.1' and not requirement_specifier.version.startswith('bc'):
raise RequirementException('can only install panda3d 1.8.1/bc1/bc2')
panda_install_dir = os.path.join(robustus.cache, 'panda3d-%s' % requirement_specifier.version)
def in_cache():
return os.path.isfile(os.path.join(panda_install_dir, 'lib/panda3d.py'))
if not in_cache() and not ignore_index:
cwd = os.getcwd()
panda3d_tgz = None
panda3d_archive_name = None
try:
panda3d_tgz = robustus.download('panda3d', requirement_specifier.version)
panda3d_archive_name = unpack(panda3d_tgz)
logging.info('Builduing panda3d')
os.chdir(panda3d_archive_name)
# link bullet into panda dependencies dir
bullet_installations = glob.glob(os.path.join(robustus.env, 'lib/bullet-*'))
if len(bullet_installations) > 0:
bullet_dir = bullet_installations[0]
if sys.platform.startswith('darwin'):
panda_thirdparty_dir = 'thirdparty/darwin-libs-a'
elif sys.platform.startswith('linux'):
panda_thirdparty_dir = 'thirdparty/linux-libs-x64'
else:
raise RequirementException('unsupported platform ' + sys.platform)
os.mkdir('thirdparty')
os.mkdir(panda_thirdparty_dir)
os.mkdir(os.path.join(panda_thirdparty_dir, 'bullet'))
ln(os.path.join(bullet_dir, 'include/bullet'),
os.path.join(panda_thirdparty_dir, 'bullet/include'))
ln(os.path.join(bullet_dir, 'lib'),
os.path.join(panda_thirdparty_dir, 'bullet/lib'))
make_panda_options = ['--nothing',
'--use-python',
'--use-direct',
'--use-bullet',
'--use-zlib',
'--use-png',
'--use-jpeg',
'--use-tiff',
'--use-freetype',
'--use-x11',
'--use-gl',
'--use-nvidiacg',
'--use-pandatool',
'--use-tinydisplay',
'--threads', '4']
if sys.platform.startswith('darwin'):
make_panda_options += ['--use-cocoa']
os.environ['CC'] = 'gcc'
os.environ['CXX'] = 'g++'
makepanda_cmd = [robustus.python_executable, 'makepanda/makepanda.py'] + make_panda_options
# command takes much time and output very long, so run_shell isn't used
retcode = subprocess.call(makepanda_cmd)
if retcode != 0:
raise RequirementException('panda3d build failed')
# copy panda3d files to cache
shutil.rmtree(panda_install_dir, ignore_errors=True)
os.mkdir(panda_install_dir)
subprocess.call('cp -R built/lib %s' % panda_install_dir, shell=True)
subprocess.call('cp -R built/bin %s' % panda_install_dir, shell=True)
subprocess.call('cp -R built/include %s' % panda_install_dir, shell=True)
subprocess.call('cp -R built/direct %s' % panda_install_dir, shell=True)
subprocess.call('cp -R built/pandac %s' % panda_install_dir, shell=True)
subprocess.call('cp -R built/models %s' % panda_install_dir, shell=True)
subprocess.call('cp -R built/etc %s' % panda_install_dir, shell=True)
finally:
safe_remove(panda3d_tgz)
safe_remove(panda3d_archive_name)
os.chdir(cwd)
if in_cache():
# install panda3d to virtualenv
libdir = os.path.join(robustus.env, 'lib/panda3d')
shutil.rmtree(libdir, ignore_errors=True)
os.mkdir(libdir)
env_etcdir = os.path.join(robustus.env, 'etc')
if not os.path.isdir(env_etcdir):
os.mkdir(env_etcdir)
etcdir = os.path.join(env_etcdir, 'panda3d')
shutil.rmtree(etcdir, ignore_errors=True)
os.mkdir(etcdir)
run_shell('cp -r -p %s/lib/* %s/' % (panda_install_dir, libdir), shell=True)
run_shell('cp -r -p %s/direct %s/' % (panda_install_dir, libdir), shell=True)
run_shell('cp -r -p %s/pandac %s/' % (panda_install_dir, libdir), shell=True)
run_shell('cp -r -p %s/etc/* %s/' % (panda_install_dir, etcdir), shell=True)
# modify rpath of libs
libdir = os.path.abspath(libdir)
if sys.platform.startswith('darwin'):
libs = glob.glob(os.path.join(libdir, '*.dylib'))
else:
libs = glob.glob(os.path.join(libdir, '*.so'))
for lib in libs:
fix_rpath(robustus, robustus.env, lib, libdir)
prc_dir_setup = "import os; os.environ['PANDA_PRC_DIR'] = '%s'" % etcdir
write_file(os.path.join(robustus.env, 'lib/python2.7/site-packages/panda3d.pth'),
'w',
'%s\n%s\n' % (libdir, prc_dir_setup))
# patch panda prc file
with open(os.path.join(etcdir, 'Config.prc'), 'a') as f:
extra_options = []
extra_options.append("# enable antialiasing\n"
"framebuffer-multisample 1\n"
"multisamples 4\n")
extra_options.append("# disable panda3d transform caching to avoid memory leak in bullet bindings\n"
"garbage-collect-states 0\n")
extra_options.append("# enable software rendering as fallback\n"
"aux-display p3tinydisplay\n")
f.write('\n'.join(extra_options))
else:
raise RequirementException('can\'t find panda3d-%s in robustus cache' % requirement_specifier.version)
def install(robustus, requirement_specifier, rob_file, ignore_index):
assert requirement_specifier.name == 'ros_overlay'
packages = requirement_specifier.version.split(',')
cwd = os.getcwd()
try:
env_source = os.path.join(robustus.env, 'bin/activate')
_ros_dep_init(env_source, robustus)
# NOTE: If ROS is not installed, the following returns an empty string.
def get_ros_install_dir(env_source):
ret_code, output = run_shell('. "%s" && python -c "import ros ; print ros.__file__"' % env_source, shell=True, return_output=True)
if ret_code != 0:
logging.info('get_ros_install_dir() failed: ret_code is %d: %s' % (ret_code, output))
return ''
if len(output.splitlines()) != 1:
logging.info('get_ros_install_dir() failed: Too many lines in output: %s' % output)
return ''
output_dirname = os.path.dirname(output)
ros_install_dir = os.path.abspath(os.path.join(output_dirname, os.pardir, os.pardir, os.pardir, os.pardir))
if not os.path.isdir(ros_install_dir):
logging.info('get_ros_install_dir() failed: ros_install_dir not a directory: %s' % ros_install_dir)
return ''
return ros_install_dir
ros_install_dir = get_ros_install_dir(env_source)
req_name = "ros-installed-overlay"
ver_hash = requirement_specifier.version_hash()
logging.info('Hashing ROS overlay on (robustus.env, ver_hash, ros_install_dir) = ("%s", "%s", "%s")' % (robustus.env, ver_hash, ros_install_dir))
req_hash = ros_utils.hash_path(robustus.env, ver_hash, ros_install_dir)
overlay_install_folder = os.path.join(robustus.cache, '%s-%s'
% (req_name, req_hash))
if not os.path.isdir(overlay_install_folder):
overlay_archive = robustus.download_compiled_archive(req_name, req_hash)
if overlay_archive:
overlay_archive_name = unpack(overlay_archive)
logging.info('Initializing compiled ROS overlay')
# install into wheelhouse
safe_move(overlay_archive_name, overlay_install_folder)
safe_remove(overlay_archive)
else:
overlay_src_folder = _make_overlay_folder(robustus, req_hash)
os.chdir(overlay_src_folder)
logging.info('Building ros overlay in %s with versions %s'
' install folder %s' % (overlay_src_folder, str(packages),
overlay_install_folder))
os.mkdir(os.path.join(overlay_src_folder, 'src'))
_get_sources(packages)
_ros_dep(env_source, robustus)
opencv_cmake_dir = _opencv_cmake_path(robustus)
ret_code = run_shell('. "%s" && export OpenCV_DIR="%s" && catkin_make_isolated'
' --install-space %s --install' %
(env_source, opencv_cmake_dir, overlay_install_folder) +
' --force-cmake --cmake-args -DCATKIN_ENABLE_TESTING=1 ',
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if ret_code != 0:
raise RequirementException('Error during catkin_make')
else:
logging.info('ROS overlay cached %s' % overlay_install_folder)
add_source_ref(robustus, os.path.join(overlay_install_folder, 'setup.sh'))
except RequirementException:
if robustus.settings['debug']:
logging.info('Not removing folder %s due to debug flag.' % overlay_src_folder)
else:
shutil.rmtree(overlay_src_folder, ignore_errors=True)
shutil.rmtree(overlay_install_folder, ignore_errors=True)
raise
finally:
os.chdir(cwd)
def install(robustus, requirement_specifier, rob_file, ignore_index):
'''
Opencv has a lot of cmake flags. Here are some examples to play with:
brew configuration for opencv 2.4.9:
'-DCMAKE_OSX_DEPLOYMENT_TARGET=',
'-DBUILD_ZLIB=OFF',
'-DBUILD_TIFF=OFF',
'-DBUILD_PNG=OFF',
'-DBUILD_OPENEXR=OFF',
'-DBUILD_JASPER=OFF',
'-DBUILD_JPEG=OFF',
'-DJPEG_INCLUDE_DIR=/usr/local/opt/jpeg/include',
'-DJPEG_LIBRARY=/usr/local/opt/jpeg/lib/libjpeg.dylib',
'-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.6_1/Frameworks/Python.framework/Versions/2.7/Python',
'-DPYTHON_INCLUDE_DIR=/usr/local/Cellar/python/2.7.6_1/Frameworks/Python.framework/Versions/2.7/Headers',
'-DBUILD_TESTS=OFF',
'-DBUILD_PERF_TESTS=OFF',
'-DBUILD_opencv_java=OFF',
'-DWITH_QT=OFF',
'-DWITH_TBB=OFF',
'-DWITH_FFMPEG=OFF',
'-DWITH_OPENEXR=OFF',
'-DWITH_CUDA=OFF',
'-DWITH_OPENCL=OFF',
'-DENABLE_SSSE3=ON',
'-DENABLE_SSE41=ON',
'-DENABLE_SSE42=ON',
'-DENABLE_AVX=ON',
'''
# Make sure numpy is installed - to avoid weird error when bindings
# are silently not generated.
if not check_module_available(robustus.env, 'numpy'):
raise RequirementException('numpy is required for opencv')
if platform.linux_distribution()[0] == 'CentOS':
# linking opencv for CentOs
logging.info('Linking opencv for CentOS')
os.symlink('/usr/lib64/python2.7/site-packages/cv2.so', os.path.join(robustus.env, 'lib/python2.7/site-packages/cv2.so'))
os.symlink('/usr/lib64/python2.7/site-packages/cv.py', os.path.join(robustus.env, 'lib/python2.7/site-packages/cv.py'))
else:
cv_install_dir = os.path.join(robustus.cache, 'OpenCV-%s' % requirement_specifier.version)
cv2so = os.path.join(cv_install_dir, 'lib/python2.7/site-packages/cv2.so')
def in_cache():
return os.path.isfile(cv2so)
if not in_cache() and not ignore_index:
cwd = os.getcwd()
opencv_archive = None
opencv_archive_name = None
try:
opencv_archive = robustus.download_compiled_archive('OpenCV', requirement_specifier.version)
if opencv_archive is not None:
opencv_archive_name = unpack(opencv_archive)
logging.info('Initializing compiled OpenCV')
# install into wheelhouse
safe_move(opencv_archive_name, cv_install_dir)
else:
opencv_archive = robustus.download('OpenCV', requirement_specifier.version)
opencv_archive_name = unpack(opencv_archive)
logging.info('Building OpenCV')
cv_build_dir = os.path.join(opencv_archive_name, 'build')
if not os.path.isdir(cv_build_dir):
os.mkdir(cv_build_dir)
os.chdir(cv_build_dir)
opencv_cmake_call_args = [
'cmake',
'../',
'-DPYTHON_EXECUTABLE=%s' % robustus.python_executable,
'-DBUILD_NEW_PYTHON_SUPPORT=ON',
'-DBUILD_TESTS=OFF',
'-DBUILD_PERF_TESTS=OFF',
'-DBUILD_DOCS=OFF',
'-DBUILD_opencv_apps=OFF',
'-DBUILD_opencv_java=OFF',
'-DWITH_CUDA=OFF',
'-DCMAKE_INSTALL_PREFIX=%s' % cv_install_dir]
if sys.platform.startswith('darwin'):
python_lib_path = subprocess.check_output(['python-config', '--prefix']).strip()
opencv_cmake_call_args.append('-DPYTHON_LIBRARY=%s/Python' % python_lib_path)
opencv_cmake_call_args.append('-DPYTHON_INCLUDE_DIR=%s/Headers' % python_lib_path)
run_shell(opencv_cmake_call_args,
verbose=robustus.settings['verbosity'] >= 1)
retcode = run_shell(['make', '-j4'], verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('OpenCV build failed')
# install into wheelhouse
if not os.path.isdir(cv_install_dir):
os.mkdir(cv_install_dir)
retcode = run_shell(['make', 'install'], verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('OpenCV installation failed')
finally:
safe_remove(opencv_archive)
safe_remove(opencv_archive_name)
os.chdir(cwd)
if in_cache():
# fix rpath for all dynamic libraries of cv2
all_cv_dlibs = os.path.abspath(os.path.join(cv_install_dir, 'lib'))
if sys.platform.startswith('darwin'):
libs = glob.glob(os.path.join(all_cv_dlibs, '*.dylib'))
else:
libs = glob.glob(os.path.join(all_cv_dlibs, '*.so'))
for lib in libs:
fix_rpath(robustus, robustus.env, lib, cv_install_dir)
if in_cache():
logging.info('Copying OpenCV cv2.so to virtualenv')
cp(os.path.join(cv_install_dir, 'lib/python2.7/site-packages/*'),
os.path.join(robustus.env, 'lib/python2.7/site-packages'))
# fix rpath for cv2
cv2lib = os.path.join(robustus.env, 'lib/python2.7/site-packages/cv2.so')
fix_rpath(robustus, robustus.env, cv2lib, cv_install_dir)
else:
raise RequirementException('can\'t find OpenCV-%s in robustus cache' % requirement_specifier.version)
def install(robustus, requirement_specifier, rob_file, ignore_index):
ver, dist = requirement_specifier.version.split('.')
if platform.machine() == 'armv7l':
# specific code to link ros on bstem
ros_install_dir = os.path.join(
'/opt/bstem/bstem.ros',
'ros-install-%s' % requirement_specifier.version)
if os.path.isdir(ros_install_dir):
# check distro
if (ver == 'hydro' and dist == 'ros_comm'):
# Add ROS settings to activate file
logging.info('Using ROS system install %s' % ros_install_dir)
add_source_ref(robustus,
os.path.join(ros_install_dir, 'setup.sh'))
return
else:
logging.warn(
'armv7l only uses hydro.ros_comm as a ROS system install.\n'
)
else:
logging.warn('No suitable ROS system install found.\n')
# check distro
if ver != 'hydro':
logging.warn(
'Robustus is only tested to install ROS hydro.\n'
'Still, it will try to install required distribution "%s"' %
requirement_specifier.version)
# install dependencies, may throw
robustus.execute([
'install', 'catkin_pkg==0.2.2', 'rosinstall==0.6.30',
'rosinstall_generator==0.1.4', 'wstool==0.0.4', 'empy==3.3.2',
'rosdep==0.10.27', 'sip'
])
ros_src_dir = os.path.join(robustus.env,
'ros-src-%s' % requirement_specifier.version)
req_name = 'ros-install-%s' % requirement_specifier.version
req_hash = ros_utils.hash_path(robustus.env)
ros_install_dir = os.path.join(robustus.cache,
'%s-%s' % (req_name, req_hash))
def in_cache():
return os.path.isdir(ros_install_dir)
rosdep = _install_ros_deps(robustus)
try:
cwd = os.getcwd()
logging.info('ROS install dir %s' % ros_install_dir)
# download and install compiled non-system ROS or, if necessary, build ROS
if not in_cache() and not ignore_index:
ros_archive = robustus.download_compiled_archive(
req_name, req_hash)
if ros_archive:
ros_archive_name = unpack(ros_archive)
logging.info(
'Initializing compiled ROS in Robustus wheelhouse')
# install into wheelhouse
safe_move(ros_archive_name, ros_install_dir)
safe_remove(ros_archive)
else:
logging.info('Building ROS in Robustus wheelhouse')
if not os.path.isdir(ros_src_dir):
os.makedirs(ros_src_dir)
os.chdir(ros_src_dir)
rosinstall_generator = os.path.join(
robustus.env, 'bin/rosinstall_generator')
retcode = run_shell(
rosinstall_generator + ' %s --rosdistro %s' % (dist, ver) +
' --deps --wet-only > %s-%s-wet.rosinstall' % (dist, ver),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException(
'Failed to generate rosinstall file')
wstool = os.path.join(robustus.env, 'bin/wstool')
retcode = run_shell(
wstool + ' init -j2 src %s-%s-wet.rosinstall' %
(dist, ver),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('Failed to build ROS')
# resolve dependencies
if sys.platform.startswith('darwin'):
ros_os_param = ''
else:
ros_os_param = '--os=ubuntu:%s' % _get_distribution()
retcode = run_shell(
rosdep +
' install -r --from-paths src --ignore-src --rosdistro %s -y %s'
% (ver, ros_os_param),
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
if platform.machine() == 'armv7l':
# Due to the lack of LISP machine for ARM we expect some failures
logging.info(
"No LISP on ARM. Expected not all dependencies to be installed."
)
else:
raise RequirementException(
'Failed to resolve ROS dependencies')
# create catkin workspace
py_activate_file = os.path.join(robustus.env, 'bin',
'activate')
catkin_make_isolated = os.path.join(
ros_src_dir, 'src/catkin/bin/catkin_make_isolated')
retcode = run_shell(
'. ' + py_activate_file + ' && ' + catkin_make_isolated +
' --install-space %s --install' % ros_install_dir,
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException(
'Failed to create catkin workspace for ROS')
logging.info('Removing ROS source/build directory %s' %
ros_src_dir)
os.chdir(
ros_install_dir
) # NOTE: If this directory is not accessible, something is wrong.
shutil.rmtree(ros_src_dir, ignore_errors=False)
else:
logging.info('Using ROS from cache %s' % ros_install_dir)
os.chdir(cwd)
# Add ROS settings to activate file
add_source_ref(robustus, os.path.join(ros_install_dir, 'setup.sh'))
except RequirementException:
os.chdir(cwd)
if robustus.settings['debug']:
logging.info('Not removing folder %s due to debug flag.' %
ros_src_dir)
logging.info('Not removing folder %s due to debug flag.' %
ros_install_dir)
else:
shutil.rmtree(ros_src_dir, ignore_errors=True)
shutil.rmtree(ros_install_dir, ignore_errors=True)
raise
def install(robustus, requirement_specifier, rob_file, ignore_index):
'''
Opencv has a lot of cmake flags. Here are some examples to play with:
brew configuration for opencv 2.4.9:
'-DCMAKE_OSX_DEPLOYMENT_TARGET=',
'-DBUILD_ZLIB=OFF',
'-DBUILD_TIFF=OFF',
'-DBUILD_PNG=OFF',
'-DBUILD_OPENEXR=OFF',
'-DBUILD_JASPER=OFF',
'-DBUILD_JPEG=OFF',
'-DJPEG_INCLUDE_DIR=/usr/local/opt/jpeg/include',
'-DJPEG_LIBRARY=/usr/local/opt/jpeg/lib/libjpeg.dylib',
'-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.6_1/Frameworks/Python.framework/Versions/2.7/Python',
'-DPYTHON_INCLUDE_DIR=/usr/local/Cellar/python/2.7.6_1/Frameworks/Python.framework/Versions/2.7/Headers',
'-DBUILD_TESTS=OFF',
'-DBUILD_PERF_TESTS=OFF',
'-DBUILD_opencv_java=OFF',
'-DWITH_QT=OFF',
'-DWITH_TBB=OFF',
'-DWITH_FFMPEG=OFF',
'-DWITH_OPENEXR=OFF',
'-DWITH_CUDA=OFF',
'-DWITH_OPENCL=OFF',
'-DENABLE_SSSE3=ON',
'-DENABLE_SSE41=ON',
'-DENABLE_SSE42=ON',
'-DENABLE_AVX=ON',
'''
# Make sure numpy is installed - to avoid weird error when bindings
# are silently not generated.
if not check_module_available(robustus.env, 'numpy'):
raise RequirementException('numpy is required for opencv')
if platform.linux_distribution()[0] == 'CentOS':
# linking opencv for CentOs
logging.info('Linking opencv for CentOS')
os.symlink('/usr/lib64/python2.7/site-packages/cv2.so', os.path.join(robustus.env, 'lib/python2.7/site-packages/cv2.so'))
os.symlink('/usr/lib64/python2.7/site-packages/cv.py', os.path.join(robustus.env, 'lib/python2.7/site-packages/cv.py'))
else:
cv_install_dir = os.path.join(robustus.cache, 'OpenCV-%s' % requirement_specifier.version)
cv2so = os.path.join(cv_install_dir, 'lib/python2.7/site-packages/cv2.so')
def in_cache():
return os.path.isfile(cv2so)
versions_to_fix_rpath = ['2.4.7', '2.4.8', '2.4.9']
if not in_cache() and not ignore_index:
cwd = os.getcwd()
opencv_archive = None
opencv_archive_name = None
try:
opencv_archive = robustus.download_compiled_archive('OpenCV', requirement_specifier.version)
if opencv_archive is not None:
opencv_archive_name = unpack(opencv_archive)
logging.info('Initializing compiled OpenCV')
# install into wheelhouse
safe_move(opencv_archive_name, cv_install_dir)
else:
opencv_archive = robustus.download('OpenCV', requirement_specifier.version)
opencv_archive_name = unpack(opencv_archive)
logging.info('Building OpenCV')
cv_build_dir = os.path.join(opencv_archive_name, 'build')
if not os.path.isdir(cv_build_dir):
os.mkdir(cv_build_dir)
os.chdir(cv_build_dir)
opencv_cmake_call_args = [
'cmake',
'../',
'-DPYTHON_EXECUTABLE=%s' % robustus.python_executable,
'-DBUILD_NEW_PYTHON_SUPPORT=ON',
'-DBUILD_TESTS=OFF',
'-DBUILD_PERF_TESTS=OFF',
'-DBUILD_DOCS=OFF',
'-DBUILD_opencv_apps=OFF',
'-DBUILD_opencv_java=OFF',
'-DWITH_CUDA=OFF',
'-DCMAKE_INSTALL_PREFIX=%s' % cv_install_dir]
if sys.platform.startswith('darwin'):
python_lib_path = subprocess.check_output(['python-config', '--prefix']).strip()
opencv_cmake_call_args.append('-DPYTHON_LIBRARY=%s/Python' % python_lib_path)
opencv_cmake_call_args.append('-DPYTHON_INCLUDE_DIR=%s/Headers' % python_lib_path)
run_shell(opencv_cmake_call_args,
verbose=robustus.settings['verbosity'] >= 1)
retcode = run_shell(['make', '-j4'], verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('OpenCV build failed')
# install into wheelhouse
if not os.path.isdir(cv_install_dir):
os.mkdir(cv_install_dir)
retcode = run_shell(['make', 'install'], verbose=robustus.settings['verbosity'] >= 1)
if retcode != 0:
raise RequirementException('OpenCV installation failed')
finally:
safe_remove(opencv_archive)
safe_remove(opencv_archive_name)
os.chdir(cwd)
if in_cache() and requirement_specifier.version in versions_to_fix_rpath:
# fix rpath for all dynamic libraries of cv2
all_cv_dlibs = os.path.abspath(os.path.join(cv_install_dir, 'lib'))
if sys.platform.startswith('darwin'):
libs = glob.glob(os.path.join(all_cv_dlibs, '*.dylib'))
else:
libs = glob.glob(os.path.join(all_cv_dlibs, '*.so'))
for lib in libs:
fix_rpath(robustus, robustus.env, lib, cv_install_dir)
if in_cache():
logging.info('Copying OpenCV cv2.so to virtualenv')
cp(os.path.join(cv_install_dir, 'lib/python2.7/site-packages/*'),
os.path.join(robustus.env, 'lib/python2.7/site-packages'))
if requirement_specifier.version in versions_to_fix_rpath:
# fix rpath for cv2
cv2lib = os.path.join(robustus.env, 'lib/python2.7/site-packages/cv2.so')
fix_rpath(robustus, robustus.env, cv2lib, cv_install_dir)
else:
raise RequirementException('can\'t find OpenCV-%s in robustus cache' % requirement_specifier.version)
def install(robustus, requirement_specifier, rob_file, ignore_index):
if requirement_specifier.version != "1.9.1" and requirement_specifier.version != "bc1":
raise RequirementException("can only install pygame 1.9.1/bc1")
if sys.platform.startswith("darwin"):
subprocess.call([robustus.pip_executable, "install", "-U", "pyobjc-core"])
subprocess.call([robustus.pip_executable, "install", "-U", "pyobjc"])
print "#####################"
print "You are on OSX"
print "Make sure you have SDL installed"
print "The easiest way to achieve this is using brew:"
print " brew install sdl sdl_image sdl_mixer sdl_ttf portmidi"
print "#####################"
pygame_cache_dir = os.path.join(robustus.cache, "pygame-%s" % requirement_specifier.version)
def in_cache():
return os.path.isfile(os.path.join(pygame_cache_dir, "pygame/__init__.py"))
if not in_cache() and not ignore_index:
cwd = os.getcwd()
pygame_archive = None
pygame_archive_name = None
try:
pygame_archive = robustus.download("pygame", requirement_specifier.version)
pygame_archive_name = unpack(pygame_archive)
# Pygame asks to proceed without smpeg,
# megahack to avoid asking to continue
logging.info("Builduing pygame")
os.chdir(pygame_archive_name)
config_unix_py = "config_unix.py"
config_unix_py_source = open(config_unix_py).read()
with open(config_unix_py, "w") as f:
f.write(
config_unix_py_source.replace("def confirm(message):", "def confirm(message):\n" " return 1\n")
)
# one more megahack to avoid problem with linux/videodev.h
# http://stackoverflow.com/questions/5842235/linux-videodev-h-no-such-file-or-directory-opencv-on-ubuntu-11-04
camera_h = "src/camera.h"
camera_h_source = open(camera_h).read()
with open(camera_h, "w") as f:
f.write(camera_h_source.replace("linux/videodev.h", "libv4l1-videodev.h"))
run_shell([robustus.python_executable, "setup.py", "build"], verbose=robustus.settings["verbosity"] >= 1)
# under build there will be platform specific dir, e.g. lib.linux-x86_64-2.7
# inside pygame will reside, copy it to robustus cache
glob_res = glob.glob("build/lib*")
if len(glob_res) == 0:
raise RequirementException("failed to build pygame-%s" % requirement_specifier.version)
pygame_dir = os.path.join(os.getcwd(), glob_res[0], "pygame")
if os.path.isdir(pygame_cache_dir):
shutil.rmtree(pygame_cache_dir)
os.mkdir(pygame_cache_dir)
shutil.copytree(pygame_dir, os.path.join(pygame_cache_dir, "pygame"))
finally:
safe_remove(pygame_archive)
safe_remove(pygame_archive_name)
os.chdir(cwd)
if in_cache():
# install pygame to virtualenv
pygame_install_dir = os.path.join(robustus.env, "lib/python2.7/site-packages")
installation_path = os.path.join(pygame_install_dir, "pygame")
if os.path.exists(installation_path):
shutil.rmtree(installation_path)
shutil.copytree(os.path.join(pygame_cache_dir, "pygame"), installation_path)
else:
raise RequirementException("can't find pygame-%s in robustus cache" % requirement_specifier.version)
def install(robustus, requirement_specifier, rob_file, ignore_index):
cwd = os.getcwd()
os.chdir(robustus.cache)
install_dir = os.path.join(robustus.cache,
'protobuf-%s' % requirement_specifier.version)
# try to download precompiled protobuf from the remote cache first
if not os.path.isdir(install_dir) and not ignore_index:
protobuf_archive = robustus.download_compiled_archive(
'protobuf', requirement_specifier.version)
if protobuf_archive is not None:
unpack(protobuf_archive)
logging.info('Initializing compiled protobuf')
# install into wheelhouse
if not os.path.exists(install_dir):
raise RequirementException(
"Failed to unpack precompiled protobuf archive")
if not os.path.isdir(install_dir) and not ignore_index:
archive_name = 'protobuf-%s.tar.gz' % requirement_specifier.version
if os.path.exists(archive_name):
safe_remove(archive_name)
# move sources to a folder in order to use a clean name for installation
src_dir = 'protobuf-%s' % requirement_specifier.version
if os.path.exists(src_dir):
safe_remove(src_dir)
run_shell([
'wget',
'https://protobuf.googlecode.com/svn/rc/%s' % (archive_name, )
],
verbose=robustus.settings['verbosity'] >= 1)
run_shell(['tar', 'zxvf', archive_name],
verbose=robustus.settings['verbosity'] >= 1)
if os.path.exists(src_dir + '_src'):
safe_remove(src_dir + '_src')
shutil.move(src_dir, src_dir + '_src')
src_dir += '_src'
os.chdir(src_dir)
if os.path.exists(install_dir):
safe_remove(install_dir)
os.mkdir(install_dir)
retcode = run_shell([
'./configure', '--disable-shared', 'CFLAGS=-fPIC',
'CXXFLAGS=-fPIC', '--prefix', install_dir
],
verbose=robustus.settings['verbosity'] >= 1)
if retcode:
raise RequirementException(
'Failed to configure protobuf compilation')
retcode = run_shell('make',
shell=True,
verbose=robustus.settings['verbosity'] >= 1)
if retcode:
raise RequirementException('Failed compile protobuf')
retcode = run_shell('make install', shell=True)
if retcode:
raise RequirementException('Failed install protobuf')
os.chdir(robustus.cache)
shutil.rmtree(src_dir)
venv_install_folder = os.path.join(robustus.env, 'protobuf')
if os.path.exists(venv_install_folder):
safe_remove(venv_install_folder)
shutil.copytree(install_dir, venv_install_folder)
executable_path = os.path.join(install_dir, 'bin', 'protoc')
ln(executable_path,
os.path.join(robustus.env, 'bin', 'protoc'),
force=True)
os.chdir(cwd)
# now install python part
robustus.install_through_wheeling(requirement_specifier, rob_file,
ignore_index)
def install(robustus, requirement_specifier, rob_file, ignore_index):
if requirement_specifier.version != '1.9.1' and requirement_specifier.version != 'bc1':
raise RequirementException('can only install pygame 1.9.1/bc1')
if sys.platform.startswith('darwin'):
subprocess.call([robustus.pip_executable, 'install', '-U', 'pyobjc-core'])
subprocess.call([robustus.pip_executable, 'install', '-U', 'pyobjc'])
print "#####################"
print "You are on OSX"
print "Make sure you have SDL installed"
print "The easiest way to achieve this is using brew:"
print " brew install sdl sdl_image sdl_mixer sdl_ttf portmidi"
print "#####################"
pygame_cache_dir = os.path.join(robustus.cache, 'pygame-%s' % requirement_specifier.version)
def in_cache():
return os.path.isfile(os.path.join(pygame_cache_dir, 'pygame/__init__.py'))
if not in_cache() and not ignore_index:
cwd = os.getcwd()
pygame_archive = None
pygame_archive_name = None
try:
pygame_archive = robustus.download('pygame', requirement_specifier.version)
pygame_archive_name = unpack(pygame_archive)
# Pygame asks to proceed without smpeg,
# megahack to avoid asking to continue
logging.info('Builduing pygame')
os.chdir(pygame_archive_name)
config_unix_py = 'config_unix.py'
config_unix_py_source = open(config_unix_py).read()
with open(config_unix_py, 'w') as f:
f.write(config_unix_py_source.replace('def confirm(message):',
'def confirm(message):\n'
' return 1\n'))
# one more megahack to avoid problem with linux/videodev.h
# http://stackoverflow.com/questions/5842235/linux-videodev-h-no-such-file-or-directory-opencv-on-ubuntu-11-04
camera_h = 'src/camera.h'
camera_h_source = open(camera_h).read()
with open(camera_h, 'w') as f:
f.write(camera_h_source.replace('linux/videodev.h',
'libv4l1-videodev.h'))
run_shell([robustus.python_executable, 'setup.py', 'build'],
verbose=robustus.settings['verbosity'] >= 1)
# under build there will be platform specific dir, e.g. lib.linux-x86_64-2.7
# inside pygame will reside, copy it to robustus cache
glob_res = glob.glob('build/lib*')
if len(glob_res) == 0:
raise RequirementException('failed to build pygame-%s' % requirement_specifier.version)
pygame_dir = os.path.join(os.getcwd(), glob_res[0], 'pygame')
if os.path.isdir(pygame_cache_dir):
shutil.rmtree(pygame_cache_dir)
os.mkdir(pygame_cache_dir)
shutil.copytree(pygame_dir, os.path.join(pygame_cache_dir, 'pygame'))
finally:
safe_remove(pygame_archive)
safe_remove(pygame_archive_name)
os.chdir(cwd)
if in_cache():
# install pygame to virtualenv
pygame_install_dir = os.path.join(robustus.env, 'lib/python2.7/site-packages')
installation_path = os.path.join(pygame_install_dir, 'pygame')
if os.path.exists(installation_path):
shutil.rmtree(installation_path)
shutil.copytree(os.path.join(pygame_cache_dir, 'pygame'),
installation_path)
else:
raise RequirementException('can\'t find pygame-%s in robustus cache' % requirement_specifier.version)
