def purge(node, parameters): package = node.get_package_name() version = node.get_version() logging.debug("Cleaning headers and cmakefiles %s" % package) node.remove_scripts_headers() node.remove_cmakefiles() logging.debug("Cleaning download %s" % package) uncompress_directory = node.get_download_directory() utils.tryremove_dir(uncompress_directory) original_directory = node.get_original_directory() utils.tryremove_dir(original_directory) for plat in platforms: if not node.get_exclude_from_clean(): logging.debug("Cleaning install %s" % package) workspace = node.get_workspace(plat) utils.tryremove_dir(workspace) build_modes = node.get_build_modes() for build_mode in build_modes: logging.debug("Cleaning build %s" % package) build_directory = node.get_build_directory(plat, build_mode) utils.tryremove_dir(build_directory) # finish well return True
def compilation(node, parameters, compiler_replace_maps): package = node.get_package_name() package_norm = node.get_package_name_norm() version = node.get_version() artifacts_dir = parameters.rootdir artifacts_dir = utils.get_norm_path(artifacts_dir) artifacts_dir = artifacts_dir.replace('\\', '/') cmake3p_dir = parameters.prefix cmake3p_dir = utils.get_norm_path(cmake3p_dir) cmake3p_dir = cmake3p_dir.replace('\\', '/') cmakelib_dir = search_cmakelib() package_upper = node.get_package_name_norm_upper() parms = node.parameters build_modes = node.get_build_modes() for plat, build_mode in product(platforms, build_modes): workspace = node.get_workspace(plat) build_directory = os.path.join( os.getcwd(), node.get_build_directory(plat, build_mode)) utils.trymkdir(build_directory) with utils.working_directory(build_directory): # get generator and platform info for compiler_c, compiler_cpp, generator, _, _, env_modified, _ in node.compiler_iterator( plat, compiler_replace_maps): logging.info('-- compilation mode: %s plat: %s' % (build_mode, plat)) ############# 1. prepare vars if build_mode.lower() == 'debug': try: env_modified[ 'CFLAGS'] = '%s -g -O0 -D_DEBUG -DDEBUG' % env_modified[ 'CFLAGS'] except KeyError: env_modified['CFLAGS'] = '-g -O0 -D_DEBUG -DDEBUG' try: env_modified[ 'CPPFLAGS'] = '%s -g -O0 -D_DEBUG -DDEBUG' % env_modified[ 'CPPFLAGS'] except KeyError: env_modified['CPPFLAGS'] = '-g -O0 -D_DEBUG -DDEBUG' elif build_mode.lower() == 'relwithdebinfo': try: env_modified[ 'CFLAGS'] = '%s -g -O2 -DNDEBUG' % env_modified[ 'CFLAGS'] except KeyError: env_modified['CFLAGS'] = '-g -O2 -DNDEBUG' try: env_modified[ 'CPPFLAGS'] = '%s -g -O2 -DNDEBUG' % env_modified[ 'CPPFLAGS'] except KeyError: env_modified['CPPFLAGS'] = '-g -O2 -DNDEBUG' elif build_mode.lower() == 'release': # default packages assume came in release pass install_directory = os.path.join(os.getcwd(), '..', workspace, node.get_base_folder(), plat) utils.trymkdir(install_directory) cores = utils.detect_ncpus() half_cores = cores / 2 env_modified['CORES'] = str(cores) env_modified['HALF_CORES'] = str(half_cores) env_modified['GTC_PREFIX'] = parameters.prefix env_modified['CMAKELIB_URL'] = CMAKELIB_URL env_modified['BUILD_MODE'] = str(build_mode) env_modified['HTTP_URL_NPSERVER'] = HTTP_URL_NPSERVER env_modified['SOURCES'] = os.path.abspath( os.path.join('..', node.get_download_directory())) env_modified['CMAKI_DIR'] = cmakelib_dir env_modified['SELFHOME'] = install_directory ################# # remove cmake3p of node node.remove_cmake3p(cmake3p_dir) # show env vars node.show_environment_vars(env_modified) # remove CMakeCache.txt for avoid problems when # change of generator utils.tryremove('CMakeCache.txt') utils.tryremove('cmake_install.cmake') utils.tryremove('install_manifest.txt') utils.tryremove_dir('CMakeFiles') ################# generator_extra = '' if generator is not None: generator_extra = '-G"%s"' % generator cmakelib_dir = parameters.cmakefiles cmakelib_dir = cmakelib_dir.replace('\\', '/') cmake_prefix_path = parameters.third_party_dir cmake_prefix_path = cmake_prefix_path.replace('\\', '/') build_directory = build_directory.replace('\\', '/') # resolve replace maps compiler_replace_resolved = {} for var, value in compiler_replace_maps.iteritems(): newvalue = value newvalue = newvalue.replace('$PLATFORM', plat) compiler_replace_resolved[var] = newvalue # begin definitions cmake try: cmake_definitions_list_original = parms[ 'cmake_definitions'] cmake_definitions_list = [] for define in cmake_definitions_list_original: # TODO: resolver tus variables directas e indirectas (de dependencias) define = define.replace('$%s_HOME' % package_norm, install_directory) # apply replaces cmake_definitions_list.append( utils.apply_replaces(define, compiler_replace_resolved)) except KeyError: cmake_definitions_list = [] # add cflags and cppflags to cmake_definitions try: cmake_definitions_list.append('CMAKE_C_FLAGS="%s"' % env_modified['CFLAGS']) except KeyError: pass try: cmake_definitions_list.append('CMAKE_CXX_FLAGS="%s"' % env_modified['CPPFLAGS']) except KeyError: pass definitions_extra = '' for definition in cmake_definitions_list: definitions_extra += ' -D%s' % definition # end definitions cmake if (not 'CMAKE_TOOLCHAIN_FILE' in env_modified) or ( not env_modified['CMAKE_TOOLCHAIN_FILE']) or ( env_modified['CMAKE_TOOLCHAIN_FILE'] == "no cross compile"): cmake_toolchain_file_filepath = '' else: cmake_toolchain_file_filepath = ' -DCMAKE_TOOLCHAIN_FILE="{}"'.format( env_modified['CMAKE_TOOLCHAIN_FILE']) cmake_prefix = node.get_cmake_prefix() cmake_configure = 'cmake %s %s -DARTIFACTS_PATH="%s" -DCMAKE_MODULE_PATH=%s -DCMAKI_PATH=%s -DCMAKE_BUILD_TYPE=%s -DAVOID_USE_HTTP=1 -DINSTALL_SIMPLE=1 -DCMAKE_PREFIX_PATH=%s -DPACKAGE=%s -DPACKAGE_UPPER=%s -DPACKAGE_VERSION=%s -DPACKAGE_BUILD_DIRECTORY=%s -DCMAKI_COMPILER=%s -DCMAKI_PLATFORM=%s %s %s' % ( generator_extra, cmake_prefix, artifacts_dir, cmakelib_dir, cmakelib_dir, build_mode, cmake_prefix_path, package, package_upper, version, build_directory, get_identifier('COMPILER'), get_identifier('ALL'), definitions_extra, cmake_toolchain_file_filepath) target = node.get_cmake_target() if target is not None: cmake_build = 'cmake --build . --target %s --config %s' % ( target, build_mode) else: cmake_build = 'cmake --build . --config %s' % (build_mode) env_modified['CMAKE_CONFIGURE'] = cmake_configure.replace( r'"', r"'") env_modified['CMAKE_BUILD'] = cmake_build.replace(r'"', r"'") ########## 2. execute executed_build_script = False if utils.is_windows(): for build_script in ['.build.cmd', 'build.cmd']: if os.path.exists(build_script): # execute manual build script node.ret += abs( utils.safe_system( '%s %s %s %s %s %s' % (build_script, install_directory, package, version, plat, build_mode), env=env_modified)) executed_build_script = True else: for build_script in ['.build.sh', 'build.sh']: if os.path.exists(build_script): with open(build_script, 'r') as f: content = f.read() # show vars node.show_environment_vars(env_modified) node.ret += abs( utils.safe_system( 'chmod +x %s && ./%s %s %s %s %s %s' % (build_script, build_script, install_directory, package, version, plat, build_mode), env=env_modified)) executed_build_script = True if not executed_build_script: logging.debug('configure command: %s' % cmake_configure) ret = utils.safe_system(cmake_configure, env=env_modified) if (ret == 0): node.ret += abs( utils.safe_system(cmake_build, env=env_modified)) else: logging.warning('Configuration failed. See log: %s' % parameters.log) node.ret += abs(ret) ######## 3. manual install # post-install logging.debug('begin post-install') for bc in node.get_post_install(): chunks = [x.strip() for x in bc.split(' ') if x] if (len(chunks) != 2) and (len(chunks) != 3): raise Exception( 'Invalid value in post_install: %s. Expected [source pattern destiny]' % bc) source_folder = os.path.join(build_directory, os.path.dirname(chunks[0])) install_directory_chunk = os.path.join( install_directory, chunks[1]) pattern = os.path.basename(chunks[0]) logging.debug( 'copy %s/%s to %s' % (source_folder, pattern, install_directory_chunk)) # create directory if not exists utils.trymkdir(install_directory_chunk) p = pipeline.make_pipe() # begin if len(chunks) == 3: p = pipeline.find(source_folder, 99)(p) else: p = pipeline.find(source_folder, 0)(p) p = pipeline.grep_basename(pattern)(p) p = pipeline.copy(source_folder, install_directory_chunk)(p) p = pipeline.debug('copied ')(p) # end pipeline.end_pipe()(p) logging.debug('end post-install') if parameters.fast: logging.debug( 'skipping for because is in fast mode: "compilation"') break # finish well return True
def run_tests(node, parameters, compiler_replace_maps, unittests): oldcwd = os.getcwd() artifacts_dir = parameters.rootdir artifacts_dir = utils.get_norm_path(artifacts_dir) artifacts_dir = artifacts_dir.replace('\\', '/') cmakelib_dir = parameters.cmakefiles cmakelib_dir = utils.get_norm_path(cmakelib_dir) cmakelib_dir = cmakelib_dir.replace('\\', '/') cmake3p_dir = parameters.prefix cmake3p_dir = utils.get_norm_path(cmake3p_dir) cmake3p_dir = cmake3p_dir.replace('\\', '/') cmake_prefix = parameters.prefix cmake_prefix = utils.get_norm_path(cmake_prefix) cmake_prefix = cmake_prefix.replace('\\', '/') cmake_third_party_dir = parameters.third_party_dir cmake_third_party_dir = utils.get_norm_path(cmake_third_party_dir) cmake_third_party_dir = cmake_third_party_dir.replace('\\', '/') package = node.get_package_name() package_upper = node.get_package_name_norm_upper() version = node.get_version() packing = node.is_packing() if not packing: logging.warning("No need run_tests, because wasn't generated a package") return 0 # prepare unittests # can be a file or content unittest_value = node.get_unittest() if unittest_value is not None: build_modes = node.get_build_modes() for plat, build_mode in product(platforms, build_modes): builddir = node.get_build_directory(plat, build_mode) path_test = os.path.join(builddir, build_unittests_foldername) utils.trymkdir(path_test) # is is a file unittest_path = os.path.join(builddir, unittest_value) if os.path.isfile(unittest_path): with open(unittest_path, 'rt') as f: unittest_value = f.read() with open(os.path.join(path_test, 'main.cpp'), 'wt') as f: f.write(unittest_value) if parameters.fast: logging.debug('skipping for because is in fast mode: "prepare"') break else: logging.warning('[%s] No test present.' % package) folder_3rdparty = parameters.third_party_dir output_3rdparty = os.path.join(folder_3rdparty, node.get_base_folder()) build_modes = node.get_build_modes() for plat, build_mode in product(platforms, reversed(build_modes)): for compiler_c, compiler_cpp, generator, _, _, env_modified, _ in node.compiler_iterator(plat, compiler_replace_maps): # verify md5sum workspace = node.get_workspace(plat) utils.trymkdir(workspace) with utils.working_directory(workspace): prefix_package = os.path.join(parameters.prefix, '%s.tar.gz' % workspace) prefix_package_md5 = os.path.join(output_3rdparty, '%s.md5' % workspace) if os.path.exists(prefix_package) and os.path.exists(prefix_package_md5): with open(prefix_package_md5, 'rt') as f: md5sum = f.read().strip() try: logging.debug("expected md5: %s" % md5sum) for line in utils.get_stdout('cmake -E md5sum %s' % prefix_package, env_modified, 'cmake'): if len(line) > 0: # md5sum filename chunks = line.split(' ') chunks = filter(None, chunks) assert(len(chunks) > 0) md5sum_real = chunks[0] logging.debug("real md5: %s" % md5sum_real) if (md5sum != md5sum_real): logging.error('Error en generated md5sum file!!!') logging.error('Expected: %s' % md5sum) logging.error('Found: %s' % md5sum_real) # add error to node node.ret += 1 except utils.NotFoundProgram: logging.info('can\'t verify md5 because not found cmake') else: logging.warning('Skipping verification md5 because don\'t exists package or md5') logging.info('running unittests. Build mode: %s Platform: %s' % (build_mode, plat)) # OJO con borrar cmake3p, se borra la marca # node.remove_cmake3p( cmake3p_dir ) builddir = os.path.join(oldcwd, node.get_build_directory(plat, build_mode)) logging.info('Using builddir %s' % builddir) unittest_folder = os.path.join(builddir, build_unittests_foldername) unittest_found = os.path.join(unittest_folder, 'main.cpp') unittest_found = unittest_found.replace('\\', '/') unittest_root = os.path.join(oldcwd, build_unittests_foldername) if os.path.exists(unittest_found): logging.info('Search cmakelib in %s' % cmakelib_dir) if os.path.isdir(os.path.join(cmakelib_dir)): with utils.working_directory(unittest_folder): generator_extra = '' if generator is not None: generator_extra = '-G"%s"' % generator find_packages = [] find_packages.append(package) for dep in node.get_depends_raw(): package_name = dep.get_package_name() find_packages.append(package_name) find_packages_str = ';'.join(find_packages) # remove CMakeCache.txt for avoid problems when # change of generator utils.tryremove('CMakeCache.txt') utils.tryremove('cmake_install.cmake') utils.tryremove('install_manifest.txt') utils.tryremove_dir('CMakeFiles') ''' TODO: refactor: prefix = DEPENDS_PATH (cmake3p) (artifacts) cmakefiles = CMAKI_PATH, CMAKE_MODULE_PATH (cmakelib) third-party-dir = CMAKE_PREFIX_PATH (directorio artifacts/cmaki_find_package) (3rdparty) rootdir = ARTIFACTS_PATH, es la base de donde esta build.py (cmaki_generator) (scripts de generacion) tambien podria ser CMAKI_PWD CMAKI_INSTALL: se espera tener instalado el cmaki_identifier ''' cmd = 'cmake %s %s -DARTIFACTS_PATH="%s" -DCMAKI_COMPILER="%s" -DCMAKI_PLATFORM="%s" -DCMAKE_MODULE_PATH="%s" -DPACKAGE="%s" -DPACKAGE_UPPER="%s" -DCMAKE_BUILD_TYPE="%s" -DAVOID_USE_HTTP=1 -DINSTALL_SIMPLE=1 -DCMAKE_PREFIX_PATH="%s" -DUNITTEST_PATH="%s" -DDEPENDS_PATH="%s" -DFIND_PACKAGES="%s" -DCMAKI_DEBUG=TRUE && cmake --build . --config %s --target install && ctest . -C %s --output-on-failure -VV' % (unittest_root, generator_extra, artifacts_dir, get_identifier('COMPILER'), get_identifier('ALL'), cmakelib_dir, package, package_upper, build_mode, cmake_third_party_dir, unittest_found, cmake_prefix, find_packages_str, build_mode, build_mode) ret = utils.safe_system(cmd, env=env_modified) node.ret += abs(ret) if ret != 0: unittests[ '%s - %s' % (package, version) ] = 'ERROR: Fail test' else: unittests[ '%s - %s' % (package, version) ] = 'OK: Pass test' else: unittests[ '%s - %s' % (package, version) ] = 'WARN: No cmakelib available' else: unittests[ '%s - %s' % (package, version) ] = 'WARN: No unittest found' # successful return True
def prepare(node, parameters, compiler_replace_maps): package = node.get_package_name() environment = node.get_first_environment(compiler_replace_maps) # source folder source_dir = os.path.join(os.getcwd(), package) utils.trymkdir(source_dir) # # generate versions.cmake node.generate_3rdpartyversion(parameters.prefix) # generate .build.sh / .build.cmd if is defined in yaml node.get_generate_custom_script(source_dir) # generate find.script / find.cmd node.generate_scripts_headers(compiler_replace_maps) # read root CMakeLists.txt with open('CMakeLists.txt', 'rt') as f: content_cmakelists = f.read() # remove packages before for plat in platforms: prefix_package = os.path.join(parameters.prefix, '%s.tar.gz' % node.get_workspace(plat)) prefix_package_cmake = os.path.join(parameters.prefix, '%s-cmakelib-%s.tar.gz' % (node.get_base_folder(), sys.platform)) prefix_folder_cmake = os.path.join(parameters.third_party_dir, node.get_base_folder()) logging.debug("preremoving package %s" % prefix_package) logging.debug("preremoving package cmakefiles %s" % prefix_package_cmake) logging.debug("preremoving folder cmakefiles %s" % prefix_folder_cmake) utils.tryremove(prefix_package) utils.tryremove(prefix_package_cmake) utils.tryremove_dir(prefix_folder_cmake) # run_tests or packing build_modes = node.get_build_modes() for plat, build_mode in product(platforms, build_modes): logging.info('Preparing mode %s - %s' % (plat, build_mode)) workspace = node.get_workspace(plat) build_directory = os.path.join(os.getcwd(), node.get_build_directory(plat, build_mode)) install_base_directory = os.path.join(os.getcwd(), workspace, node.get_base_folder()) utils.trymkdir(build_directory) # download source and prepare in build_directory node.prepare_third_party(build_directory, compiler_replace_maps) # copy source files to build logging.debug('Copy sources to build: %s -> %s' % (source_dir, build_directory)) utils.copy_folder_recursive(source_dir, build_directory) # before copy files with utils.working_directory(build_directory): for bc in node.get_before_copy(): chunks = [x.strip() for x in bc.split(' ') if x] if(len(chunks) != 2): raise Exception('Invalid value in before_copy: %s' % bc) logging.debug('Copy "%s" to "%s"' % (chunks[0], chunks[1])) shutil.copy2(chunks[0], chunks[1]) # if have cmakelists, insert root cmakelists header cmake_prefix = node.get_cmake_prefix() build_cmakelist = os.path.join(build_directory, cmake_prefix, 'CMakeLists.txt') if os.path.exists(build_cmakelist) and (not node.has_custom_script(source_dir)): with open(build_cmakelist, 'rt') as f: content_cmakelists_package = f.read() with open(build_cmakelist, 'wt') as f: f.write('%s\n' % content_cmakelists) f.write('%s\n' % content_cmakelists_package) if parameters.fast: logging.debug('skipping for because is in fast mode: "prepare"') break # finish well return True