def extract_autotools_deps(outlines, srctree, acfile=None): import shlex import oe.package values = {} inherits = [] # FIXME this mapping is very thin progmap = {'flex': 'flex-native', 'bison': 'bison-native', 'm4': 'm4-native'} progclassmap = {'gconftool-2': 'gconf', 'pkg-config': 'pkgconfig'} ignoredeps = ['gcc-runtime', 'glibc', 'uclibc'] pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)[),].*') lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*') progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)\]?[),].*') dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') # Build up lib library->package mapping shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data) libdir = tinfoil.config_data.getVar('libdir', True) base_libdir = tinfoil.config_data.getVar('base_libdir', True) libpaths = list(set([base_libdir, libdir])) libname_re = re.compile('^lib(.+)\.so.*$') pkglibmap = {} for lib, item in shlib_providers.iteritems(): for path, pkg in item.iteritems(): if path in libpaths: res = libname_re.match(lib) if res: libname = res.group(1) if not libname in pkglibmap: pkglibmap[libname] = pkg[0] else: logger.debug('unable to extract library name from %s' % lib) # Now turn it into a library->recipe mapping recipelibmap = {} pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) for libname, pkg in pkglibmap.iteritems(): try: with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: for line in f: if line.startswith('PN:'): recipelibmap[libname] = line.split(':', 1)[-1].strip() break except IOError as ioe: if ioe.errno == 2: logger.warn('unable to find a pkgdata file for package %s' % pkg) else: raise # Since a configure.ac file is essentially a program, this is only ever going to be # a hack unfortunately; but it ought to be enough of an approximation if acfile: srcfiles = [acfile] else: srcfiles = RecipeHandler.checkfiles(srctree, ['configure.ac', 'configure.in']) pcdeps = [] deps = [] unmapped = [] unmappedlibs = [] with open(srcfiles[0], 'r') as f: for line in f: if 'PKG_CHECK_MODULES' in line: res = pkg_re.search(line) if res: res = dep_re.findall(res.group(1)) if res: pcdeps.extend([x[0] for x in res]) inherits.append('pkgconfig') if line.lstrip().startswith('AM_GNU_GETTEXT'): inherits.append('gettext') elif 'AC_CHECK_PROG' in line or 'AC_PATH_PROG' in line: res = progs_re.search(line) if res: for prog in shlex.split(res.group(1)): prog = prog.split()[0] progclass = progclassmap.get(prog, None) if progclass: inherits.append(progclass) else: progdep = progmap.get(prog, None) if progdep: deps.append(progdep) else: if not prog.startswith('$'): unmapped.append(prog) elif 'AC_CHECK_LIB' in line: res = lib_re.search(line) if res: lib = res.group(1) libdep = recipelibmap.get(lib, None) if libdep: deps.append(libdep) else: if libdep is None: if not lib.startswith('$'): unmappedlibs.append(lib) elif 'AC_PATH_X' in line: deps.append('libx11') if unmapped: outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(unmapped)) if unmappedlibs: outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(unmappedlibs)) outlines.append('# (this is based on recipes that have previously been built and packaged)') recipemap = read_pkgconfig_provides(tinfoil.config_data) unmapped = [] for pcdep in pcdeps: recipe = recipemap.get(pcdep, None) if recipe: deps.append(recipe) else: if not pcdep.startswith('$'): unmapped.append(pcdep) deps = set(deps).difference(set(ignoredeps)) if unmapped: outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmapped)) outlines.append('# (this is based on recipes that have previously been built and packaged)') if deps: values['DEPENDS'] = ' '.join(deps) if inherits: values['inherit'] = ' '.join(list(set(inherits))) return values
def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): import shlex import oe.package values = {} inherits = [] # FIXME this mapping is very thin progmap = {'flex': 'flex-native', 'bison': 'bison-native', 'm4': 'm4-native', 'tar': 'tar-native', 'ar': 'binutils-native'} progclassmap = {'gconftool-2': 'gconf', 'pkg-config': 'pkgconfig'} ignoredeps = ['gcc-runtime', 'glibc', 'uclibc', 'tar-native', 'binutils-native'] ignorelibs = ['socket'] pkg_re = re.compile('PKG_CHECK_MODULES\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)[),].*') lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*') progs_re = re.compile('_PROGS?\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)\]?[),].*') dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*') am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') # Build up lib library->package mapping shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data) libdir = tinfoil.config_data.getVar('libdir', True) base_libdir = tinfoil.config_data.getVar('base_libdir', True) libpaths = list(set([base_libdir, libdir])) libname_re = re.compile('^lib(.+)\.so.*$') pkglibmap = {} for lib, item in shlib_providers.iteritems(): for path, pkg in item.iteritems(): if path in libpaths: res = libname_re.match(lib) if res: libname = res.group(1) if not libname in pkglibmap: pkglibmap[libname] = pkg[0] else: logger.debug('unable to extract library name from %s' % lib) # Now turn it into a library->recipe mapping recipelibmap = {} pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) for libname, pkg in pkglibmap.iteritems(): try: with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: for line in f: if line.startswith('PN:'): recipelibmap[libname] = line.split(':', 1)[-1].strip() break except IOError as ioe: if ioe.errno == 2: logger.warn('unable to find a pkgdata file for package %s' % pkg) else: raise defines = {} def subst_defines(value): newvalue = value for define, defval in defines.iteritems(): newvalue = newvalue.replace(define, defval) if newvalue != value: return subst_defines(newvalue) return value def process_value(value): value = value.replace('[', '').replace(']', '') if value.startswith('m4_esyscmd(') or value.startswith('m4_esyscmd_s('): cmd = subst_defines(value[value.index('(')+1:-1]) try: if '|' in cmd: cmd = 'set -o pipefail; ' + cmd stdout, _ = bb.process.run(cmd, cwd=srctree, shell=True) ret = stdout.rstrip() except bb.process.ExecutionError as e: ret = '' elif value.startswith('m4_'): return None ret = subst_defines(value) if ret: ret = ret.strip('"\'') return ret # Since a configure.ac file is essentially a program, this is only ever going to be # a hack unfortunately; but it ought to be enough of an approximation if acfile: srcfiles = [acfile] else: srcfiles = RecipeHandler.checkfiles(srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) pcdeps = [] deps = [] unmapped = [] unmappedlibs = [] def process_macro(keyword, value): if keyword == 'PKG_CHECK_MODULES': res = pkg_re.search(value) if res: res = dep_re.findall(res.group(1)) if res: pcdeps.extend([x[0] for x in res]) inherits.append('pkgconfig') elif keyword in ('AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE'): inherits.append('gettext') elif keyword in ('AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL'): deps.append('intltool-native') elif keyword == 'AM_PATH_GLIB_2_0': deps.append('glib-2.0') elif keyword == 'AC_CHECK_PROG' or keyword == 'AC_PATH_PROG': res = progs_re.search(value) if res: for prog in shlex.split(res.group(1)): prog = prog.split()[0] progclass = progclassmap.get(prog, None) if progclass: inherits.append(progclass) else: progdep = progmap.get(prog, None) if progdep: deps.append(progdep) else: if not prog.startswith('$'): unmapped.append(prog) elif keyword == 'AC_CHECK_LIB': res = lib_re.search(value) if res: lib = res.group(1) if lib in ignorelibs: logger.debug('Ignoring library dependency %s' % lib) else: libdep = recipelibmap.get(lib, None) if libdep: deps.append(libdep) else: if libdep is None: if not lib.startswith('$'): unmappedlibs.append(lib) elif keyword == 'AC_PATH_X': deps.append('libx11') elif keyword == 'AC_INIT': if extravalues is not None: res = ac_init_re.match(value) if res: extravalues['PN'] = process_value(res.group(1)) pv = process_value(res.group(2)) if validate_pv(pv): extravalues['PV'] = pv elif keyword == 'AM_INIT_AUTOMAKE': if extravalues is not None: if 'PN' not in extravalues: res = am_init_re.match(value) if res: if res.group(1) != 'AC_PACKAGE_NAME': extravalues['PN'] = process_value(res.group(1)) pv = process_value(res.group(2)) if validate_pv(pv): extravalues['PV'] = pv elif keyword == 'define(': res = define_re.match(value) if res: key = res.group(2).strip('[]') value = process_value(res.group(3)) if value is not None: defines[key] = value keywords = ['PKG_CHECK_MODULES', 'AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE', 'AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL', 'AM_PATH_GLIB_2_0', 'AC_CHECK_PROG', 'AC_PATH_PROG', 'AC_CHECK_LIB', 'AC_PATH_X', 'AC_INIT', 'AM_INIT_AUTOMAKE', 'define(', ] for srcfile in srcfiles: nesting = 0 in_keyword = '' partial = '' with open(srcfile, 'r') as f: for line in f: if in_keyword: partial += ' ' + line.strip() if partial.endswith('\\'): partial = partial[:-1] nesting = nesting + line.count('(') - line.count(')') if nesting == 0: process_macro(in_keyword, partial) partial = '' in_keyword = '' else: for keyword in keywords: if keyword in line: nesting = line.count('(') - line.count(')') if nesting > 0: partial = line.strip() if partial.endswith('\\'): partial = partial[:-1] in_keyword = keyword else: process_macro(keyword, line.strip()) break if in_keyword: process_macro(in_keyword, partial) if extravalues: for k,v in extravalues.items(): if v: if v.startswith('$') or v.startswith('@') or v.startswith('%'): del extravalues[k] else: extravalues[k] = v.strip('"\'').rstrip('()') if unmapped: outlines.append('# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped)))) if unmappedlibs: outlines.append('# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs)))) outlines.append('# (this is based on recipes that have previously been built and packaged)') recipemap = read_pkgconfig_provides(tinfoil.config_data) unmapped = [] for pcdep in pcdeps: recipe = recipemap.get(pcdep, None) if recipe: deps.append(recipe) else: if not pcdep.startswith('$'): unmapped.append(pcdep) deps = set(deps).difference(set(ignoredeps)) if unmapped: outlines.append('# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmapped)) outlines.append('# (this is based on recipes that have previously been built and packaged)') if deps: values['DEPENDS'] = ' '.join(deps) if inherits: values['inherit'] = ' '.join(list(set(inherits))) return values
def extract_autotools_deps(outlines, srctree, extravalues=None, acfile=None): import shlex import oe.package values = {} inherits = [] # FIXME this mapping is very thin progmap = { 'flex': 'flex-native', 'bison': 'bison-native', 'm4': 'm4-native', 'tar': 'tar-native', 'ar': 'binutils-native' } progclassmap = {'gconftool-2': 'gconf', 'pkg-config': 'pkgconfig'} ignoredeps = [ 'gcc-runtime', 'glibc', 'uclibc', 'tar-native', 'binutils-native' ] ignorelibs = ['socket'] pkg_re = re.compile( 'PKG_CHECK_MODULES\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)[),].*') lib_re = re.compile('AC_CHECK_LIB\(\[?([a-zA-Z0-9]*)\]?, .*') progs_re = re.compile( '_PROGS?\(\[?[a-zA-Z0-9]*\]?, \[?([^,\]]*)\]?[),].*') dep_re = re.compile('([^ ><=]+)( [<>=]+ [^ ><=]+)?') ac_init_re = re.compile('AC_INIT\(([^,]+), *([^,]+)[,)].*') am_init_re = re.compile('AM_INIT_AUTOMAKE\(([^,]+), *([^,]+)[,)].*') define_re = re.compile(' *(m4_)?define\(([^,]+), *([^,]+)\)') # Build up lib library->package mapping shlib_providers = oe.package.read_shlib_providers(tinfoil.config_data) libdir = tinfoil.config_data.getVar('libdir', True) base_libdir = tinfoil.config_data.getVar('base_libdir', True) libpaths = list(set([base_libdir, libdir])) libname_re = re.compile('^lib(.+)\.so.*$') pkglibmap = {} for lib, item in shlib_providers.iteritems(): for path, pkg in item.iteritems(): if path in libpaths: res = libname_re.match(lib) if res: libname = res.group(1) if not libname in pkglibmap: pkglibmap[libname] = pkg[0] else: logger.debug('unable to extract library name from %s' % lib) # Now turn it into a library->recipe mapping recipelibmap = {} pkgdata_dir = tinfoil.config_data.getVar('PKGDATA_DIR', True) for libname, pkg in pkglibmap.iteritems(): try: with open(os.path.join(pkgdata_dir, 'runtime', pkg)) as f: for line in f: if line.startswith('PN:'): recipelibmap[libname] = line.split(':', 1)[-1].strip() break except IOError as ioe: if ioe.errno == 2: logger.warn( 'unable to find a pkgdata file for package %s' % pkg) else: raise defines = {} def subst_defines(value): newvalue = value for define, defval in defines.iteritems(): newvalue = newvalue.replace(define, defval) if newvalue != value: return subst_defines(newvalue) return value def process_value(value): value = value.replace('[', '').replace(']', '') if value.startswith('m4_esyscmd(') or value.startswith( 'm4_esyscmd_s('): cmd = subst_defines(value[value.index('(') + 1:-1]) try: if '|' in cmd: cmd = 'set -o pipefail; ' + cmd stdout, _ = bb.process.run(cmd, cwd=srctree, shell=True) ret = stdout.rstrip() except bb.process.ExecutionError as e: ret = '' elif value.startswith('m4_'): return None ret = subst_defines(value) if ret: ret = ret.strip('"\'') return ret # Since a configure.ac file is essentially a program, this is only ever going to be # a hack unfortunately; but it ought to be enough of an approximation if acfile: srcfiles = [acfile] else: srcfiles = RecipeHandler.checkfiles( srctree, ['acinclude.m4', 'configure.ac', 'configure.in']) pcdeps = [] deps = [] unmapped = [] unmappedlibs = [] def process_macro(keyword, value): if keyword == 'PKG_CHECK_MODULES': res = pkg_re.search(value) if res: res = dep_re.findall(res.group(1)) if res: pcdeps.extend([x[0] for x in res]) inherits.append('pkgconfig') elif keyword in ('AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE'): inherits.append('gettext') elif keyword in ('AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL'): deps.append('intltool-native') elif keyword == 'AM_PATH_GLIB_2_0': deps.append('glib-2.0') elif keyword == 'AC_CHECK_PROG' or keyword == 'AC_PATH_PROG': res = progs_re.search(value) if res: for prog in shlex.split(res.group(1)): prog = prog.split()[0] progclass = progclassmap.get(prog, None) if progclass: inherits.append(progclass) else: progdep = progmap.get(prog, None) if progdep: deps.append(progdep) else: if not prog.startswith('$'): unmapped.append(prog) elif keyword == 'AC_CHECK_LIB': res = lib_re.search(value) if res: lib = res.group(1) if lib in ignorelibs: logger.debug('Ignoring library dependency %s' % lib) else: libdep = recipelibmap.get(lib, None) if libdep: deps.append(libdep) else: if libdep is None: if not lib.startswith('$'): unmappedlibs.append(lib) elif keyword == 'AC_PATH_X': deps.append('libx11') elif keyword == 'AC_INIT': if extravalues is not None: res = ac_init_re.match(value) if res: extravalues['PN'] = process_value(res.group(1)) pv = process_value(res.group(2)) if validate_pv(pv): extravalues['PV'] = pv elif keyword == 'AM_INIT_AUTOMAKE': if extravalues is not None: if 'PN' not in extravalues: res = am_init_re.match(value) if res: if res.group(1) != 'AC_PACKAGE_NAME': extravalues['PN'] = process_value(res.group(1)) pv = process_value(res.group(2)) if validate_pv(pv): extravalues['PV'] = pv elif keyword == 'define(': res = define_re.match(value) if res: key = res.group(2).strip('[]') value = process_value(res.group(3)) if value is not None: defines[key] = value keywords = [ 'PKG_CHECK_MODULES', 'AM_GNU_GETTEXT', 'AM_GLIB_GNU_GETTEXT', 'GETTEXT_PACKAGE', 'AC_PROG_INTLTOOL', 'IT_PROG_INTLTOOL', 'AM_PATH_GLIB_2_0', 'AC_CHECK_PROG', 'AC_PATH_PROG', 'AC_CHECK_LIB', 'AC_PATH_X', 'AC_INIT', 'AM_INIT_AUTOMAKE', 'define(', ] for srcfile in srcfiles: nesting = 0 in_keyword = '' partial = '' with open(srcfile, 'r') as f: for line in f: if in_keyword: partial += ' ' + line.strip() if partial.endswith('\\'): partial = partial[:-1] nesting = nesting + line.count('(') - line.count(')') if nesting == 0: process_macro(in_keyword, partial) partial = '' in_keyword = '' else: for keyword in keywords: if keyword in line: nesting = line.count('(') - line.count(')') if nesting > 0: partial = line.strip() if partial.endswith('\\'): partial = partial[:-1] in_keyword = keyword else: process_macro(keyword, line.strip()) break if in_keyword: process_macro(in_keyword, partial) if extravalues: for k, v in extravalues.items(): if v: if v.startswith('$') or v.startswith('@') or v.startswith( '%'): del extravalues[k] else: extravalues[k] = v.strip('"\'').rstrip('()') if unmapped: outlines.append( '# NOTE: the following prog dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmapped)))) if unmappedlibs: outlines.append( '# NOTE: the following library dependencies are unknown, ignoring: %s' % ' '.join(list(set(unmappedlibs)))) outlines.append( '# (this is based on recipes that have previously been built and packaged)' ) recipemap = read_pkgconfig_provides(tinfoil.config_data) unmapped = [] for pcdep in pcdeps: recipe = recipemap.get(pcdep, None) if recipe: deps.append(recipe) else: if not pcdep.startswith('$'): unmapped.append(pcdep) deps = set(deps).difference(set(ignoredeps)) if unmapped: outlines.append( '# NOTE: unable to map the following pkg-config dependencies: %s' % ' '.join(unmapped)) outlines.append( '# (this is based on recipes that have previously been built and packaged)' ) if deps: values['DEPENDS'] = ' '.join(deps) if inherits: values['inherit'] = ' '.join(list(set(inherits))) return values