def _build_make(self, build_only_project):
utils.call(
'make -j%i %s' % (
self.env.c.parallel_build,
build_only_project if build_only_project else ''),
cwd=self.env.env_path,
shell=True)
def db_open_dump(self):
"""Open the latest database dump in a text editor"""
uncompressed_target = join(
self.db_path, 'config_{project_name}.sql'.format(
project_name=self.config.project_name))
if os.path.isfile(self.config.editor_path):
utils.call([self.config.editor_path, uncompressed_target], bg=True)
def db_view_gis(self):
"""Open database in a GIS GUI tool (if applicable)"""
if self.db_backend.name == 'sqlite':
utils.call(
[self.config.spatialite_gis_path,
self.db_backend.conn_info['path']],
bg=True)
else:
raise NotImplementedError("Not implemented for this backend")
def run_cpp_tests(self, suite_args):
self.update_environment_for_cpp_tests()
builder = synthesepy.build.get_builder(self.env)
args = [builder.get_cmake_tool_path('ctest')]
if self.env.c.verbose:
args.append('-V')
if self.KNOWN_FAILURES:
failing_tests = '|'.join(['test_{0}_{1}Test'.format(path, test) for
path, test in self.KNOWN_FAILURES])
args.extend(['-E', failing_tests])
utils.call(args, cwd=self.env.env_path)
return True
def _call_spatialite(self, cmd, shell=True, **kwargs):
# TODO: this should use the compiled spatialite.
# Warning: shell=False is required on Linux, otherwise it launches the
# interpreter and it hangs.
cmd = [self.env.config.spatialite_path] + cmd
return utils.call(cmd, shell=False, **kwargs)
def _call_spatialite(self, cmd, shell=True, **kwargs):
# TODO: this should use the compiled spatialite.
# Warning: shell=False is required on Linux, otherwise it launches the
# interpreter and it hangs.
cmd = [self.env.config.spatialite_path] + cmd
return utils.call(cmd, shell=False, **kwargs)
def _install_cmake(self):
self.cmake_path = None
cmake_executable = 'cmake' + self.env.platform_exe_suffix
CMAKE_VERSION = '2.8.5'
if utils.find_executable(cmake_executable):
cmake_version = subprocess.Popen(
[cmake_executable, '--version'], stdout=subprocess.PIPE
).communicate()[0].strip()
if self.extract_version_as_int(cmake_version, r".* (\d+).(\d+).(\d+)") >= \
self.extract_version_as_int(CMAKE_VERSION):
log.info('Found system cmake')
return
else:
log.info('Warning, Your installed version of CMAKE is tool old.')
log.info('Installing cmake')
CMAKE_URL_BASE = 'http://www.cmake.org/files/v2.8/'
if self.env.platform == 'win':
url = '%scmake-%s-win32-x86.zip' % (CMAKE_URL_BASE, CMAKE_VERSION)
self._download(url, 'ef536e5148aacf559735df893b40a1f4')
created_dir = self._extract(url, self.env.c.thirdparty_dir)
self.cmake_path = join(self.env.c.thirdparty_dir, created_dir, 'bin')
else:
cmake_install_path = join(self.env.c.thirdparty_dir, 'cmake')
cmake_src_path = join(
self.env.c.thirdparty_dir, 'cmake-%s' % CMAKE_VERSION)
self.cmake_path = join(cmake_install_path, 'bin')
if (os.path.isdir(cmake_src_path) and
os.path.isdir(cmake_install_path)):
return
url = '%scmake-%s.tar.gz' % (CMAKE_URL_BASE, CMAKE_VERSION)
self._download(url, '3c5d32cec0f4c2dc45f4c2e84f4a20c5')
created_dir = self._extract(url, self.env.c.thirdparty_dir)
log.info('Building cmake')
utils.RemoveDirectory(cmake_install_path)
cmake_src = join(self.env.c.thirdparty_dir, created_dir)
utils.call(
[join(cmake_src, 'configure'), '--prefix=' +
cmake_install_path],
cwd=cmake_src)
utils.call(
['make', '-j%i' % self.env.c.parallel_build, 'install'],
cwd=cmake_src)
def _mysql_command(self, command, extra_opts='', input='', table=None):
self._setup_path()
default_args = {
'port': '3306',
}
args = default_args
args.update(self.conn_info.data)
args.update(dict(
command=command,
extra_opts=extra_opts,
table=table if table else '',
))
cmd = ('{command} {extra_opts} -u{user} -p{passwd} -h{host} '
'-P{port} {db} {table}'.format(**args))
if input is not None:
return utils.call(cmd, input=input)
return utils.call(cmd)
def _mysql_command(self, command, extra_opts='', input='', table=None):
self._setup_path()
default_args = {
'port': '3306',
}
args = default_args
args.update(self.conn_info.data)
args.update(
dict(
command=command,
extra_opts=extra_opts,
table=table if table else '',
))
cmd = ('{command} {extra_opts} -u{user} -p{passwd} -h{host} '
'-P{port} {db} {table}'.format(**args))
if input is not None:
return utils.call(cmd, input=input)
return utils.call(cmd)
def _install_cmake(self):
self.cmake_path = None
cmake_executable = "cmake" + self.env.platform_exe_suffix
CMAKE_VERSION = "2.8.5"
if utils.find_executable(cmake_executable):
cmake_version = (
subprocess.Popen([cmake_executable, "--version"], stdout=subprocess.PIPE).communicate()[0].strip()
)
if self.extract_version_as_int(cmake_version, r".* (\d+).(\d+).(\d+)") >= self.extract_version_as_int(
CMAKE_VERSION
):
log.info("Found system cmake")
return
else:
log.info("Warning, Your installed version of CMAKE is tool old.")
log.info("Installing cmake")
CMAKE_URL_BASE = "http://www.cmake.org/files/v2.8/"
if self.env.platform == "win":
url = "%scmake-%s-win32-x86.zip" % (CMAKE_URL_BASE, CMAKE_VERSION)
self._download(url, "ef536e5148aacf559735df893b40a1f4")
created_dir = self._extract(url, self.env.c.thirdparty_dir)
self.cmake_path = join(self.env.c.thirdparty_dir, created_dir, "bin")
else:
cmake_install_path = join(self.env.c.thirdparty_dir, "cmake")
cmake_src_path = join(self.env.c.thirdparty_dir, "cmake-%s" % CMAKE_VERSION)
self.cmake_path = join(cmake_install_path, "bin")
if os.path.isdir(cmake_src_path) and os.path.isdir(cmake_install_path):
return
url = "%scmake-%s.tar.gz" % (CMAKE_URL_BASE, CMAKE_VERSION)
self._download(url, "3c5d32cec0f4c2dc45f4c2e84f4a20c5")
created_dir = self._extract(url, self.env.c.thirdparty_dir)
log.info("Building cmake")
utils.RemoveDirectory(cmake_install_path)
cmake_src = join(self.env.c.thirdparty_dir, created_dir)
utils.call([join(cmake_src, "configure"), "--prefix=" + cmake_install_path], cwd=cmake_src)
utils.call(["make", "-j%i" % self.env.c.parallel_build, "install"], cwd=cmake_src)
def run(env, args):
builder = synthesepy.build.get_builder(env)
builder.check_debian_package_requirements(['doxygen', 'graphviz'])
target_dir = join(env.env_path, 'doc')
# TODO: should we keep it to save time?
utils.RemoveDirectory(target_dir)
os.makedirs(target_dir)
doxyfile = join(env.source_path, 'doc', 'Doxyfile.doxyfile')
doxyfile_generated = _generate_doxyfile(env, doxyfile, target_dir)
# Copy include
source_include = join(env.source_path, 'doc', 'include')
target_include = join(target_dir, 'html', 'include')
utils.RemoveDirectory(target_include)
assert not os.path.isdir(target_include)
log.debug('Copying include: %r -> %r', source_include, target_include)
shutil.copytree(source_include, target_include)
utils.call(['doxygen', doxyfile_generated], cwd=os.path.dirname(doxyfile))
def _run_devenv(self, build_project=None):
# TODO: this should be extracted from system config
default_vs_path = (os.environ['ProgramFiles'] +
'\\Microsoft Visual Studio 9.0\\')
utils.append_paths_to_environment('PATH', [
default_vs_path + 'Common7\\IDE',
])
if build_project:
# TODO: this should be extracted from system config
default_sdk_path = 'C:\\Program Files\\Microsoft SDKs\\Windows\\v6.0A'
utils.append_paths_to_environment('PATH', [
default_vs_path + 'Common7\\Tools',
default_vs_path + 'VC\\BIN',
default_vs_path + 'VC\\VCPackages',
default_sdk_path + 'bin',
])
utils.append_paths_to_environment('INCLUDE', [
default_vs_path + 'VC\\ATLMFC\\INCLUDE',
default_vs_path + 'VC\\INCLUDE',
default_sdk_path + 'include',
])
utils.append_paths_to_environment('LIB', [
default_vs_path + 'VC\\ATLMFC\\LIB',
default_vs_path + 'VC\\LIB',
default_sdk_path + 'lib',
])
args = ['devenv.com', 'synthese3.sln']
if build_project:
args.extend([
'/build', self.env.build_type, '/project', build_project])
utils.call(
args,
cwd=self.env.env_path)
def _run_devenv(self, build_project=None):
# TODO: this should be extracted from system config
default_vs_path = os.environ["ProgramFiles"] + "\\Microsoft Visual Studio 9.0\\"
utils.append_paths_to_environment("PATH", [default_vs_path + "Common7\\IDE"])
if build_project:
# TODO: this should be extracted from system config
default_sdk_path = "C:\\Program Files\\Microsoft SDKs\\Windows\\v6.0A"
utils.append_paths_to_environment(
"PATH",
[
default_vs_path + "Common7\\Tools",
default_vs_path + "VC\\BIN",
default_vs_path + "VC\\VCPackages",
default_sdk_path + "bin",
],
)
utils.append_paths_to_environment(
"INCLUDE",
[
default_vs_path + "VC\\ATLMFC\\INCLUDE",
default_vs_path + "VC\\INCLUDE",
default_sdk_path + "include",
],
)
utils.append_paths_to_environment(
"LIB", [default_vs_path + "VC\\ATLMFC\\LIB", default_vs_path + "VC\\LIB", default_sdk_path + "lib"]
)
args = ["devenv.com", "synthese3.sln"]
if build_project:
args.extend(["/build", self.env.build_type, "/project", build_project])
utils.call(args, cwd=self.env.env_path)
def call_command(cls, project, cmd, hide_arg=None):
command_result = CommandResult()
command_result.commandline = cmd
if hide_arg:
command_result.commandline = \
command_result.commandline.replace(hide_arg, '***')
try:
command_result.output = utils.call(
cmd, cwd=project.path, ret_output=True)
log.debug('Command output:\n%s',
'\n'.join('\t' + l for l in command_result.output.splitlines()))
except subprocess.CalledProcessError, e:
command_result.output = e.output
command_result.status = CommandResult.FAILURE
def run(env, args):
builder = synthesepy.build.get_builder(env)
builder.check_debian_package_requirements(['doxygen', 'graphviz'])
target_dir = join(env.env_path, 'doc')
# TODO: should we keep it to save time?
utils.RemoveDirectory(target_dir)
os.makedirs(target_dir)
doxyfile = join(env.source_path, 'doc', 'Doxyfile.doxyfile')
doxyfile_generated = _generate_doxyfile(env, doxyfile, target_dir)
# Copy include
source_include = join(env.source_path, 'doc', 'include')
target_include = join(target_dir, 'html', 'include')
utils.RemoveDirectory(target_include)
assert not os.path.isdir(target_include)
log.debug('Copying include: %r -> %r', source_include, target_include)
shutil.copytree(source_include, target_include)
utils.call(
['doxygen', doxyfile_generated],
cwd=os.path.dirname(doxyfile))
def root_delegate(self, command, args=[]):
if command == 'update_synthese':
install_url = args[0]
# Security check: we don't want a compromised synthese user be
# able to execute any remote script.
REQUIRED_URL_RE = '^http://ci.rcsmobility.com/~build/[\w/\.]+$'
if not re.match(REQUIRED_URL_RE, install_url):
raise Exception('The install url must match the regexp %r' %
REQUIRED_URL_RE)
if self.env.config.dummy:
return
utils.call('curl -s {0} | python'.format(install_url))
elif command in ('start_supervisor', 'stop_supervisor'):
supervisor_command = 'start' if command == 'start_supervisor' else 'stop'
utils.call([
self.config.supervisorctl_path, supervisor_command,
'synthese-{0}'.format(self.config.project_name.replace('_', '-'))])
elif command in ('start_initd', 'stop_initd'):
initd_command = 'start' if command == 'start_initd' else 'stop'
utils.call(['/etc/init.d/s3-server', initd_command])
else:
raise Exception('Unknown command %r' % command)
def run(env, args):
config = env.config
if not config.prefix:
raise Exception('Prefix is required.')
# Pass the branch from the command line, useful in Jenkins
# where we get HEAD instead of the branch
git_info = utils.GITInfo(env.source_path, config.branch)
revision_path = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
revision_path += '-{0}'.format(git_info.version)
# On Linux, lets pick a more precise name for the platform
# through lsb_release (this package must be installed)
distro_name = env.platform
if env.platform == 'lin':
try:
# TODO: use subprocess.check_output once we require Python 2.7
distro_name = subprocess.Popen(
["lsb_release", "-cs"],
stdout=subprocess.PIPE).communicate()[0]
distro_name = distro_name.rstrip()
except:
raise Exception('Failed to run lsb_release. '
'Please install it (apt-get install lsb-release)')
package_relative_dir = os.sep.join(
[env.platform, env.mode, git_info.branch, revision_path])
package_dir = join(config.packages_save_path, package_relative_dir)
if os.path.isdir(package_dir):
if config.no_package_overwrite:
log.info('Package directory already exists (%r) not building.',
package_dir)
return
try:
if not os.path.isdir(package_dir):
os.makedirs(package_dir)
except:
raise Exception('Failed to create the '
'directory in %r, cannot continue' % package_dir)
log.info('Cleaning prefix %r', config.prefix)
utils.RemoveDirectory(config.prefix)
log.info('Installing Synthese to %r', config.prefix)
builder = synthesepy.build.get_builder(env)
builder.install()
env_dir = os.path.join(config.prefix, 'share', 'synthese', 'env')
if not os.path.isdir(env_dir):
log.info('CMake install didn\'t create the environment '
'directory in %r, we do not seal it' % env_dir)
else:
with open(join(env_dir, 'sealed.txt'), 'wb') as f:
f.write('Environment sealed\n')
# Archive
ARCHIVE_NAME = 'synthese.tar.bz2'
if config.prefix_with_svnrelease:
ARCHIVE_NAME = 'synthese-relative-' + revision_path + '.tar.bz2'
archive_path = join(package_dir, ARCHIVE_NAME)
log.info('Creating archive %r', archive_path)
prefix_parent = os.path.dirname(config.prefix)
prefix_tail = os.path.basename(config.prefix)
utils.call([
'tar', '-C',
utils.to_cygwin_path(prefix_parent), '-jcf',
utils.to_cygwin_path(archive_path), '--owner=0', '--group=0',
'--numeric-owner', '--mode=go-w', prefix_tail
])
log.info('Remove the build directory in %r' % config.prefix)
utils.RemoveDirectory(config.prefix)
# Deploy script
deploy_script_path = join(package_dir, 'install_synthese.py')
source_deploy_script = join(env.source_path, 'tools', 'synthesepy',
'install_synthese.py.in')
deploy_script_content = open(source_deploy_script).read()
archive_url = (config.packages_access_url + package_relative_dir + '/' +
ARCHIVE_NAME)
deploy_script_content = deploy_script_content.replace(
'@@ARCHIVE_URL@@', archive_url).replace('@@PREFIX@@', config.prefix)
with open(deploy_script_path, 'wb') as f:
f.write(deploy_script_content)
log.debug('Deploy script written to %r', deploy_script_path)
# TODO: remove old packages to avoid filling up the disk.
# latest symlink
if env.platform != 'win':
link_name = join(package_dir, os.pardir, 'latest')
if os.path.exists(link_name):
os.unlink(link_name)
os.symlink(revision_path, link_name)
def _install_make(self):
utils.call(
'make -j%i install' % self.env.c.parallel_build,
cwd=self.env.env_path,
shell=True)
def call_project(self, command, global_args=[], args=[], sudo=False):
cmd = self.build_command_line(command, global_args, args, sudo)
utils.call(cmd, cwd=self.path)
def _install_boost(self):
self.boost_dir = None
self.boost_lib_dir = None
if self.config.boost_dir:
self.boost_dir = self.config.boost_dir
return
if self.env.platform != 'win':
# Assume we'll use the system version
return
# Boost dependencies
BZIP2_ARCHIVE = 'bzip2-1.0.6'
url = 'http://www.bzip.org/1.0.6/{0}.tar.gz'.format(BZIP2_ARCHIVE)
self._download(url, '00b516f4704d4a7cb50a1d97e6e8e15b')
created_dir = self._extract(url, self.env.c.thirdparty_dir)
ZLIB_ARCHIVE = 'zlib-1.2.8'
url = 'http://zlib.net/zlib-1.2.8.tar.gz'
self._download(url, '44d667c142d7cda120332623eab69f40')
created_dir = self._extract(url, self.env.c.thirdparty_dir)
url = 'http://switch.dl.sourceforge.net/project/boost/boost/1.42.0/boost_1_42_0.zip'
self._download(url, 'ceb78ed309c867e49dc29f60be841b64')
created_dir = self._extract(url, self.env.c.thirdparty_dir)
self.boost_dir = join(self.env.c.thirdparty_dir, created_dir)
self.boost_lib_dir = join(self.boost_dir, 'stage', 'lib')
# Patch for zlib
zlibjam = join(self.boost_dir, "libs/iostreams/build/Jamfile.v2")
s = open(zlibjam).read()
s = s.replace('gzio', '')
f = open(zlibjam, 'w')
f.write(s)
f.close()
CURRENT_BOOST_BUILD_VER = 2
boost_build_ver_path = join(
self.env.c.thirdparty_dir, 'boost_build_ver.txt')
try:
boost_build_ver = int(open(boost_build_ver_path).read())
except IOError:
boost_build_ver = 0
log.debug('Found current boost build version: %i', boost_build_ver)
if (boost_build_ver >= CURRENT_BOOST_BUILD_VER and
os.path.isdir(self.boost_lib_dir)):
return
log.info("Building Boost, this can take some times...")
utils.call(
join(self.boost_dir, 'bootstrap.bat'), cwd=self.boost_dir)
args = [join(self.boost_dir, 'bjam.exe')]
# TODO: have an option to specify the vs version.
toolset = 'msvc-9.0'
args.extend(
'toolset={toolset} {x64} release debug link=static runtime-link=static '
'threading=multi'.format(
toolset=toolset, x64='address-model=64' if self.env.c.x64 else ''
).split(' '))
args.extend(['--with-%s' % m for m in REQUIRED_BOOST_MODULES])
args.append('-sBZIP2_SOURCE={}'.format(
join(self.env.c.thirdparty_dir, BZIP2_ARCHIVE)))
args.append('-sZLIB_SOURCE={}'.format(
join(self.env.c.thirdparty_dir, ZLIB_ARCHIVE)))
utils.call(args, cwd=self.boost_dir)
open(boost_build_ver_path, 'wb').write(str(CURRENT_BOOST_BUILD_VER))
def _generate_build_system(self):
if self.config.clear_cmake_cache:
utils.maybe_remove(join(self.env.env_path, 'CMakeCache.txt'))
args = [self.get_cmake_tool_path('cmake'), self.env.source_path]
# Use ccache on Linux if available
if self.env.platform == 'lin' and utils.find_executable('ccache'):
os.environ['CXX'] = 'ccache g++'
os.environ['CC'] = 'ccache gcc'
if self.env.platform == 'win':
# TODO: This shouldn't be hardcoded.
args.extend(['-G', 'Visual Studio 9 2008' + (' Win64' if self.env.c.x64 else '')])
if self.with_mysql:
args.append('-DWITH_MYSQL=1')
if self.mysql_dir:
os.environ['MYSQL_DIR'] = self.mysql_dir
args.append('-DCMAKE_BUILD_TYPE=' + self.env.build_type)
# TODO: maybe change optimization flags in debug mode:
# -DCMAKE_CXX_FLAGS=-O0
# Add a suffix to the install dir to allow
# different SYNTHESE to run on the same server
if self.env.config.prefix_with_svnrelease:
git_info = utils.GITInfo(self.env.source_path, None)
revision_path = '-r{0}'.format(git_info.version)
self.config.prefix += revision_path
if self.config.prefix:
args.append('-DCMAKE_INSTALL_PREFIX=' + self.config.prefix)
if self.config.mysql_params:
args.append('-DSYNTHESE_MYSQL_PARAMS=' + self.config.mysql_params)
args.append('-DBOOST_VERSION=' + BOOST_VER)
env = os.environ.copy()
if self.boost_dir:
env['BOOST_ROOT'] = self.boost_dir
if self.boost_lib_dir:
env['BOOST_LIBRARYDIR'] = self.boost_lib_dir
# Enable subdirs
args.append('-DWITH_TEST:BOOL=ON')
if self.config.do_not_build_python:
args.append('-DWITH_PACKAGES:BOOL=OFF')
args.append('-DWITH_PROJECTS:BOOL=OFF')
args.append('-DWITH_TOOLS:BOOL=OFF')
args.append('-DWITH_UTILS:BOOL=OFF')
else:
args.append('-DWITH_PACKAGES:BOOL=ON')
args.append('-DWITH_PROJECTS:BOOL=ON')
args.append('-DWITH_TOOLS:BOOL=ON')
args.append('-DWITH_UTILS:BOOL=ON')
# TODO: check that Python Cygwin is not in the path?
if not os.path.isdir(self.env.env_path):
os.makedirs(self.env.env_path)
utils.call(args, cwd=self.env.env_path, env=env)
# Hack to disable incremental build on XP (it fails with:
# LINK : fatal error LNK1210: exceeded internal ILK size limit; link with /INCREMENTAL:NO)
# I didn't find a way to to this in CMakeLists.txt
# (http://www.cmake.org/pipermail/cmake/2010-February/035174.html didn't work)
if self.env.platform == 'win' and platform.release() == 'XP':
cmake_cache = join(self.env.env_path, 'CMakeCache.txt')
cache_content = open(cmake_cache).read()
cache_content = cache_content.replace(
'INCREMENTAL:YES', 'INCREMENTAL:NO')
open(cmake_cache, 'wb').write(cache_content)
def run(env, args):
config = env.config
if not config.prefix:
raise Exception('Prefix is required.')
svn_info = utils.SVNInfo(env.source_path)
revision_path = 'r{0}'.format(svn_info.version)
package_relative_dir = os.sep.join([
env.platform, env.mode, svn_info.branch, revision_path])
package_dir = join(config.packages_save_path, package_relative_dir)
if os.path.isdir(package_dir):
if config.no_package_overwrite:
log.info('Package directory already exists (%r) not building.',
package_dir)
return
try:
if not os.path.isdir(package_dir):
os.makedirs(package_dir)
except:
raise Exception('Failed to create the '
'directory in %r, cannot continue' % package_dir)
log.info('Cleaning prefix %r', config.prefix)
utils.RemoveDirectory(config.prefix)
log.info('Installing Synthese to %r', config.prefix)
builder = synthesepy.build.get_builder(env)
builder.install()
env_dir = os.path.join(config.prefix, 'share', 'synthese', 'env')
if not os.path.isdir(env_dir):
log.info('CMake install didn\'t create the environment '
'directory in %r, we do not seal it' % env_dir)
else:
with open(join(env_dir, 'sealed.txt'), 'wb') as f:
f.write('Environment sealed\n')
# Archive
ARCHIVE_NAME = 'synthese.tar.bz2'
if config.prefix_with_svnrelease:
ARCHIVE_NAME = 'synthese-relative-' + revision_path + '.tar.bz2'
archive_path = join(package_dir, ARCHIVE_NAME)
log.info('Creating archive %r', archive_path)
prefix_parent = os.path.dirname(config.prefix)
prefix_tail = os.path.basename(config.prefix)
utils.call([
'tar', '-C', utils.to_cygwin_path(prefix_parent),
'-jcf', utils.to_cygwin_path(archive_path),
'--owner=0', '--group=0',
'--numeric-owner', '--mode=go-w', prefix_tail])
log.info('Remove the build directory in %r' % config.prefix)
utils.RemoveDirectory(config.prefix)
# Deploy script
deploy_script_path = join(package_dir, 'install_synthese.py')
source_deploy_script = join(
env.source_path, 'tools', 'synthesepy', 'install_synthese.py.in')
deploy_script_content = open(source_deploy_script).read()
archive_url = (config.packages_access_url + package_relative_dir +
'/' + ARCHIVE_NAME)
deploy_script_content = deploy_script_content.replace(
'@@ARCHIVE_URL@@', archive_url).replace(
'@@PREFIX@@', config.prefix)
with open(deploy_script_path, 'wb') as f:
f.write(deploy_script_content)
log.debug('Deploy script written to %r', deploy_script_path)
# TODO: remove old packages to avoid filling up the disk.
# latest symlink
if env.platform != 'win':
link_name = join(package_dir, os.pardir, 'latest')
if os.path.exists(link_name):
os.unlink(link_name)
os.symlink(revision_path, link_name)
def ssh(self):
"""Open a ssh shell on the remote server"""
utils.call(utils.ssh_command_line(self.config))
def system_install(self):
self.create_symlink(self._get_link_path(), self.config_path)
utils.call('a2enmod proxy', shell=True)
utils.call('a2enmod proxy_http', shell=True)
utils.call('a2enmod rewrite', shell=True)
utils.call('a2enmod deflate', shell=True)
utils.call('a2enmod setenvif', shell=True)
utils.call('a2enmod headers', shell=True)
utils.call('a2enmod filter', shell=True)
utils.call('a2enmod wsgi', shell=True)
utils.call('/etc/init.d/apache2 graceful', shell=True)
def system_uninstall(self):
os.unlink(self._get_link_path())
# Note: not removing apache modules. They might be used by other things.
utils.call('/etc/init.d/apache2 graceful', shell=True)
def _install_boost(self):
self.boost_dir = None
self.boost_lib_dir = None
if self.config.boost_dir:
self.boost_dir = self.config.boost_dir
return
if self.env.platform != "win":
# Assume we'll use the system version
return
# Boost dependencies
BZIP2_ARCHIVE = "bzip2-1.0.6"
url = "http://www.bzip.org/1.0.6/{0}.tar.gz".format(BZIP2_ARCHIVE)
self._download(url, "00b516f4704d4a7cb50a1d97e6e8e15b")
created_dir = self._extract(url, self.env.c.thirdparty_dir)
ZLIB_ARCHIVE = "zlib-1.2.8"
url = "http://zlib.net/zlib-1.2.8.tar.gz"
self._download(url, "44d667c142d7cda120332623eab69f40")
created_dir = self._extract(url, self.env.c.thirdparty_dir)
url = "http://switch.dl.sourceforge.net/project/boost/boost/1.42.0/boost_1_42_0.zip"
self._download(url, "ceb78ed309c867e49dc29f60be841b64")
created_dir = self._extract(url, self.env.c.thirdparty_dir)
self.boost_dir = join(self.env.c.thirdparty_dir, created_dir)
self.boost_lib_dir = join(self.boost_dir, "stage", "lib")
# Patch for zlib
zlibjam = join(self.boost_dir, "libs/iostreams/build/Jamfile.v2")
s = open(zlibjam).read()
s = s.replace("gzio", "")
f = open(zlibjam, "w")
f.write(s)
f.close()
CURRENT_BOOST_BUILD_VER = 2
boost_build_ver_path = join(self.env.c.thirdparty_dir, "boost_build_ver.txt")
try:
boost_build_ver = int(open(boost_build_ver_path).read())
except IOError:
boost_build_ver = 0
log.debug("Found current boost build version: %i", boost_build_ver)
if boost_build_ver >= CURRENT_BOOST_BUILD_VER and os.path.isdir(self.boost_lib_dir):
return
log.info("Building Boost, this can take some times...")
utils.call(join(self.boost_dir, "bootstrap.bat"), cwd=self.boost_dir)
args = [join(self.boost_dir, "bjam.exe")]
# TODO: have an option to specify the vs version.
toolset = "msvc-9.0"
args.extend(
"toolset={toolset} {x64} release debug link=static runtime-link=static "
"threading=multi".format(toolset=toolset, x64="address-model=64" if self.env.c.x64 else "").split(" ")
)
args.extend(["--with-%s" % m for m in REQUIRED_BOOST_MODULES])
args.append("-sBZIP2_SOURCE={}".format(join(self.env.c.thirdparty_dir, BZIP2_ARCHIVE)))
args.append("-sZLIB_SOURCE={}".format(join(self.env.c.thirdparty_dir, ZLIB_ARCHIVE)))
utils.call(args, cwd=self.boost_dir)
open(boost_build_ver_path, "wb").write(str(CURRENT_BOOST_BUILD_VER))
def _generate_build_system(self):
if self.config.clear_cmake_cache:
utils.maybe_remove(join(self.env.env_path, "CMakeCache.txt"))
args = [self.get_cmake_tool_path("cmake"), self.env.source_path]
# Use ccache on Linux if available
if self.env.platform == "lin" and utils.find_executable("ccache"):
os.environ["CXX"] = "ccache g++"
os.environ["CC"] = "ccache gcc"
if self.env.platform == "win":
# TODO: This shouldn't be hardcoded.
args.extend(["-G", "Visual Studio 9 2008" + (" Win64" if self.env.c.x64 else "")])
if self.with_mysql:
args.append("-DWITH_MYSQL=1")
if self.mysql_dir:
os.environ["MYSQL_DIR"] = self.mysql_dir
args.append("-DCMAKE_BUILD_TYPE=" + self.env.build_type)
# TODO: maybe change optimization flags in debug mode:
# -DCMAKE_CXX_FLAGS=-O0
# Add a suffix to the install dir to allow
# different SYNTHESE to run on the same server
if self.env.config.prefix_with_svnrelease:
svn_info = utils.SVNInfo(self.env.source_path)
revision_path = "-r{0}".format(svn_info.version)
self.config.prefix += revision_path
if self.config.prefix:
args.append("-DCMAKE_INSTALL_PREFIX=" + self.config.prefix)
if self.config.mysql_params:
args.append("-DSYNTHESE_MYSQL_PARAMS=" + self.config.mysql_params)
args.append("-DBOOST_VERSION=" + BOOST_VER)
env = os.environ.copy()
if self.boost_dir:
env["BOOST_ROOT"] = self.boost_dir
if self.boost_lib_dir:
env["BOOST_LIBRARYDIR"] = self.boost_lib_dir
# Enable subdirs
args.append("-DWITH_TEST:BOOL=ON")
if self.config.do_not_build_python:
args.append("-DWITH_PACKAGES:BOOL=OFF")
args.append("-DWITH_PROJECTS:BOOL=OFF")
args.append("-DWITH_TOOLS:BOOL=OFF")
args.append("-DWITH_UTILS:BOOL=OFF")
else:
args.append("-DWITH_PACKAGES:BOOL=ON")
args.append("-DWITH_PROJECTS:BOOL=ON")
args.append("-DWITH_TOOLS:BOOL=ON")
args.append("-DWITH_UTILS:BOOL=ON")
# TODO: check that Python Cygwin is not in the path?
if not os.path.isdir(self.env.env_path):
os.makedirs(self.env.env_path)
utils.call(args, cwd=self.env.env_path, env=env)
# Hack to disable incremental build on XP (it fails with:
# LINK : fatal error LNK1210: exceeded internal ILK size limit; link with /INCREMENTAL:NO)
# I didn't find a way to to this in CMakeLists.txt
# (http://www.cmake.org/pipermail/cmake/2010-February/035174.html didn't work)
if self.env.platform == "win" and platform.release() == "XP":
cmake_cache = join(self.env.env_path, "CMakeCache.txt")
cache_content = open(cmake_cache).read()
cache_content = cache_content.replace("INCREMENTAL:YES", "INCREMENTAL:NO")
open(cmake_cache, "wb").write(cache_content)
def system_uninstall(self):
os.unlink(self._get_link_path())
utils.call('supervisorctl reread', shell=True)
def system_install(self):
self.create_symlink(self._get_link_path(), self.config_path)
utils.call('supervisorctl reread', shell=True)
def run(env, args):
config = env.config
if not config.prefix:
raise Exception('Prefix is required.')
# Pass the branch from the command line, useful in Jenkins
# where we get HEAD instead of the branch
git_info = utils.GITInfo(env.source_path, config.branch)
revision_path = datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
revision_path += '-{0}'.format(git_info.version)
# On Linux, lets pick a more precise name for the platform
# through lsb_release (this package must be installed)
distro_name = env.platform
if env.platform == 'lin':
try:
# TODO: use subprocess.check_output once we require Python 2.7
distro_name = subprocess.Popen(
["lsb_release", "-cs"], stdout=subprocess.PIPE).communicate()[0]
distro_name = distro_name.rstrip()
except:
raise Exception('Failed to run lsb_release. '
'Please install it (apt-get install lsb-release)')
package_relative_dir = os.sep.join([
env.platform, env.mode, git_info.branch, revision_path])
package_dir = join(config.packages_save_path, package_relative_dir)
if os.path.isdir(package_dir):
if config.no_package_overwrite:
log.info('Package directory already exists (%r) not building.',
package_dir)
return
try:
if not os.path.isdir(package_dir):
os.makedirs(package_dir)
except:
raise Exception('Failed to create the '
'directory in %r, cannot continue' % package_dir)
log.info('Cleaning prefix %r', config.prefix)
utils.RemoveDirectory(config.prefix)
log.info('Installing Synthese to %r', config.prefix)
builder = synthesepy.build.get_builder(env)
builder.install()
env_dir = os.path.join(config.prefix, 'share', 'synthese', 'env')
if not os.path.isdir(env_dir):
log.info('CMake install didn\'t create the environment '
'directory in %r, we do not seal it' % env_dir)
else:
with open(join(env_dir, 'sealed.txt'), 'wb') as f:
f.write('Environment sealed\n')
# Archive
ARCHIVE_NAME = 'synthese.tar.bz2'
if config.prefix_with_svnrelease:
ARCHIVE_NAME = 'synthese-relative-' + revision_path + '.tar.bz2'
archive_path = join(package_dir, ARCHIVE_NAME)
log.info('Creating archive %r', archive_path)
prefix_parent = os.path.dirname(config.prefix)
prefix_tail = os.path.basename(config.prefix)
utils.call([
'tar', '-C', utils.to_cygwin_path(prefix_parent),
'-jcf', utils.to_cygwin_path(archive_path),
'--owner=0', '--group=0',
'--numeric-owner', '--mode=go-w', prefix_tail])
log.info('Remove the build directory in %r' % config.prefix)
utils.RemoveDirectory(config.prefix)
# Deploy script
deploy_script_path = join(package_dir, 'install_synthese.py')
source_deploy_script = join(
env.source_path, 'tools', 'synthesepy', 'install_synthese.py.in')
deploy_script_content = open(source_deploy_script).read()
archive_url = (config.packages_access_url + package_relative_dir +
'/' + ARCHIVE_NAME)
deploy_script_content = deploy_script_content.replace(
'@@ARCHIVE_URL@@', archive_url).replace(
'@@PREFIX@@', config.prefix)
with open(deploy_script_path, 'wb') as f:
f.write(deploy_script_content)
log.debug('Deploy script written to %r', deploy_script_path)
# TODO: remove old packages to avoid filling up the disk.
# latest symlink
if env.platform != 'win':
link_name = join(package_dir, os.pardir, 'latest')
if os.path.exists(link_name):
os.unlink(link_name)
os.symlink(revision_path, link_name)