def get_recipe_upgrade_status(recipes=None):
    pkgs_list = []
    data_copy_list = []
    copy_vars = ('SRC_URI',
                 'PV',
                 'GITDIR',
                 'DL_DIR',
                 'PN',
                 'CACHE',
                 'PERSISTENT_DIR',
                 'BB_URI_HEADREVS',
                 'UPSTREAM_CHECK_COMMITS',
                 'UPSTREAM_CHECK_GITTAGREGEX',
                 'UPSTREAM_CHECK_REGEX',
                 'UPSTREAM_CHECK_URI',
                 'UPSTREAM_VERSION_UNKNOWN',
                 'RECIPE_MAINTAINER',
                 'RECIPE_NO_UPDATE_REASON',
                 'RECIPE_UPSTREAM_VERSION',
                 'RECIPE_UPSTREAM_DATE',
                 'CHECK_DATE',
            )

    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.prepare(config_only=False)

        if not recipes:
            recipes = tinfoil.all_recipe_files(variants=False)

        for fn in recipes:
            try:
                if fn.startswith("/"):
                    data = tinfoil.parse_recipe_file(fn)
                else:
                    data = tinfoil.parse_recipe(fn)
            except bb.providers.NoProvider:
                bb.note(" No provider for %s" % fn)
                continue

            unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
            if unreliable == "1":
                bb.note(" Skip package %s as upstream check unreliable" % pn)
                continue

            data_copy = bb.data.init()
            for var in copy_vars:
                data_copy.setVar(var, data.getVar(var))
            for k in data:
                if k.startswith('SRCREV'):
                    data_copy.setVar(k, data.getVar(k))

            data_copy_list.append(data_copy)

    from concurrent.futures import ProcessPoolExecutor
    with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
        pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)

    return pkgs_list
Example #2
0
 def test_parse_recipe(self):
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=False, quiet=2)
         testrecipe = 'mdadm'
         best = tinfoil.find_best_provider(testrecipe)
         if not best:
             self.fail('Unable to find recipe providing %s' % testrecipe)
         rd = tinfoil.parse_recipe_file(best[3])
         self.assertEqual(testrecipe, rd.getVar('PN'))
Example #3
0
 def test_parse_recipe_initial_datastore(self):
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=False, quiet=2)
         testrecipe = 'mdadm'
         best = tinfoil.find_best_provider(testrecipe)
         if not best:
             self.fail('Unable to find recipe providing %s' % testrecipe)
         dcopy = bb.data.createCopy(tinfoil.config_data)
         dcopy.setVar('MYVARIABLE', 'somevalue')
         rd = tinfoil.parse_recipe_file(best[3], config_data=dcopy)
         # Check we can get variable values
         self.assertEqual('somevalue', rd.getVar('MYVARIABLE'))
Example #4
0
def get_recipe_upgrade_status(recipes=None):
    pkgs_list = []
    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.prepare(config_only=False)

        if not recipes:
            recipes = tinfoil.all_recipe_files(variants=False)

        for fn in recipes:
            try:
                if fn.startswith("/"):
                    data = tinfoil.parse_recipe_file(fn)
                else:
                    data = tinfoil.parse_recipe(fn)
            except bb.providers.NoProvider:
                bb.note(" No provider for %s" % fn)
                continue

            unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
            if unreliable == "1":
                bb.note(" Skip package %s as upstream check unreliable" % pn)
                continue

            uv = get_recipe_upstream_version(data)

            pn = data.getVar('PN')
            cur_ver = uv['current_version']

            upstream_version_unknown = data.getVar('UPSTREAM_VERSION_UNKNOWN')
            if not uv['version']:
                status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"
            else:
                cmp = vercmp_string(uv['current_version'], uv['version'])
                if cmp == -1:
                    status = "UPDATE" if not upstream_version_unknown else "KNOWN_BROKEN"
                elif cmp == 0:
                    status = "MATCH" if not upstream_version_unknown else "KNOWN_BROKEN"
                else:
                    status = "UNKNOWN" if upstream_version_unknown else "UNKNOWN_BROKEN"

            next_ver = uv['version'] if uv['version'] else "N/A"
            revision = uv['revision'] if uv['revision'] else "N/A"
            maintainer = data.getVar('RECIPE_MAINTAINER')
            no_upgrade_reason = data.getVar('RECIPE_NO_UPDATE_REASON')

            pkgs_list.append((pn, status, cur_ver, next_ver, maintainer, revision, no_upgrade_reason))

    return pkgs_list
Example #5
0
def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
    """Parse the specified recipe"""
    try:
        recipefile = tinfoil.get_recipe_file(pn)
    except bb.providers.NoProvider as e:
        logger.error(str(e))
        return None
    if appends:
        append_files = tinfoil.get_file_appends(recipefile)
        if filter_workspace:
            # Filter out appends from the workspace
            append_files = [path for path in append_files if
                            not path.startswith(config.workspace_path)]
    else:
        append_files = None
    return tinfoil.parse_recipe_file(recipefile, appends, append_files)
Example #6
0
 def test_parse_recipe_copy_expand(self):
     with bb.tinfoil.Tinfoil() as tinfoil:
         tinfoil.prepare(config_only=False, quiet=2)
         testrecipe = 'mdadm'
         best = tinfoil.find_best_provider(testrecipe)
         if not best:
             self.fail('Unable to find recipe providing %s' % testrecipe)
         rd = tinfoil.parse_recipe_file(best[3])
         # Check we can get variable values
         self.assertEqual(testrecipe, rd.getVar('PN'))
         # Check that expanding a value that includes a variable reference works
         self.assertEqual(testrecipe, rd.getVar('BPN'))
         # Now check that changing the referenced variable's value in a copy gives that
         # value when expanding
         localdata = bb.data.createCopy(rd)
         localdata.setVar('PN', 'hello')
         self.assertEqual('hello', localdata.getVar('BPN'))
Example #7
0
def parse_recipe(config, tinfoil, pn, appends, filter_workspace=True):
    """Parse the specified recipe"""
    try:
        recipefile = tinfoil.get_recipe_file(pn)
    except bb.providers.NoProvider as e:
        logger.error(str(e))
        return None
    if appends:
        append_files = tinfoil.get_file_appends(recipefile)
        if filter_workspace:
            # Filter out appends from the workspace
            append_files = [
                path for path in append_files
                if not path.startswith(config.workspace_path)
            ]
    else:
        append_files = None
    try:
        rd = tinfoil.parse_recipe_file(recipefile, appends, append_files)
    except Exception as e:
        logger.error(str(e))
        return None
    return rd
Example #8
0
def get_recipe_upgrade_status(recipes=None):
    pkgs_list = []
    data_copy_list = []
    copy_vars = (
        'SRC_URI',
        'PV',
        'DL_DIR',
        'PN',
        'CACHE',
        'PERSISTENT_DIR',
        'BB_URI_HEADREVS',
        'UPSTREAM_CHECK_COMMITS',
        'UPSTREAM_CHECK_GITTAGREGEX',
        'UPSTREAM_CHECK_REGEX',
        'UPSTREAM_CHECK_URI',
        'UPSTREAM_VERSION_UNKNOWN',
        'RECIPE_MAINTAINER',
        'RECIPE_NO_UPDATE_REASON',
        'RECIPE_UPSTREAM_VERSION',
        'RECIPE_UPSTREAM_DATE',
        'CHECK_DATE',
        'FETCHCMD_bzr',
        'FETCHCMD_ccrc',
        'FETCHCMD_cvs',
        'FETCHCMD_git',
        'FETCHCMD_hg',
        'FETCHCMD_npm',
        'FETCHCMD_osc',
        'FETCHCMD_p4',
        'FETCHCMD_repo',
        'FETCHCMD_s3',
        'FETCHCMD_svn',
        'FETCHCMD_wget',
    )

    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.prepare(config_only=False)

        if not recipes:
            recipes = tinfoil.all_recipe_files(variants=False)

        for fn in recipes:
            try:
                if fn.startswith("/"):
                    data = tinfoil.parse_recipe_file(fn)
                else:
                    data = tinfoil.parse_recipe(fn)
            except bb.providers.NoProvider:
                bb.note(" No provider for %s" % fn)
                continue

            unreliable = data.getVar('UPSTREAM_CHECK_UNRELIABLE')
            if unreliable == "1":
                bb.note(" Skip package %s as upstream check unreliable" % pn)
                continue

            data_copy = bb.data.init()
            for var in copy_vars:
                data_copy.setVar(var, data.getVar(var))
            for k in data:
                if k.startswith('SRCREV'):
                    data_copy.setVar(k, data.getVar(k))

            data_copy_list.append(data_copy)

    from concurrent.futures import ProcessPoolExecutor
    with ProcessPoolExecutor(max_workers=utils.cpu_count()) as executor:
        pkgs_list = executor.map(_get_recipe_upgrade_status, data_copy_list)

    return pkgs_list
Example #9
0
def main():
    docfiles = []
    metadirs = []
    bbvars = set()
    undocumented = []
    docconf = ""
    onlydoctags = False

    # Collect and validate input
    try:
        opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"])
    except getopt.GetoptError as err:
        print('%s' % str(err))
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ('-h', '--help'):
            usage()
            sys.exit(0)
        elif o == '-d':
            if os.path.isfile(a):
                docfiles.append(a)
            else:
                print('ERROR: documentation file %s is not a regular file' % a)
                sys.exit(3)
        elif o == '-m':
            if os.path.isdir(a):
                metadirs.append(a)
            else:
                print('ERROR: meta directory %s is not a directory' % a)
                sys.exit(4)
        elif o == "-t":
            if os.path.isfile(a):
                docconf = a
        elif o == "-T":
            onlydoctags = True
        else:
            assert False, "unhandled option"

    if len(docfiles) == 0:
        print('ERROR: no docfile specified')
        usage()
        sys.exit(5)

    if len(metadirs) == 0:
        print('ERROR: no metadir specified')
        usage()
        sys.exit(6)

    if onlydoctags and docconf == "":
        print('ERROR: no docconf specified')
        usage()
        sys.exit(7)

    prog = re.compile("^[^a-z]*$")
    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.prepare(config_only=False)
        parser = bb.codeparser.PythonParser('parser', None)
        datastore = tinfoil.config_data

        def bbvars_update(data):
            if prog.match(data):
                bbvars.add(data)
            if tinfoil.config_data.getVarFlag(data, 'python'):
                try:
                    parser.parse_python(tinfoil.config_data.getVar(data))
                except bb.data_smart.ExpansionError:
                    pass
                for var in parser.references:
                    if prog.match(var):
                        bbvars.add(var)
            else:
                try:
                    expandedVar = datastore.expandWithRefs(datastore.getVar(data, False), data)
                    for var in expandedVar.references:
                        if prog.match(var):
                            bbvars.add(var)
                except bb.data_smart.ExpansionError:
                    pass

        # Use tinfoil to collect all the variable names globally
        for data in datastore:
            bbvars_update(data)

        # Collect variables from all recipes
        for recipe in tinfoil.all_recipe_files():
            print("Checking %s" % recipe)
            for data in tinfoil.parse_recipe_file(recipe):
                bbvars_update(data)

    documented_vars = collect_documented_vars(docfiles)

    # Check each var for documentation
    varlen = 0
    for v in bbvars:
        if len(v) > varlen:
            varlen = len(v)
        if not bbvar_is_documented(v, documented_vars):
            undocumented.append(v)
    undocumented.sort()
    varlen = varlen + 1

    # Report all undocumented variables
    print('Found %d undocumented bb variables (out of %d):' % (len(undocumented), len(bbvars)))
    header = '%s%s' % (str("VARIABLE").ljust(varlen), str("DOCTAG").ljust(7))
    print(header)
    print(str("").ljust(len(header), '='))
    for v in undocumented:
        doctag = bbvar_doctag(v, docconf)
        if not onlydoctags or not doctag == "":
            print('%s%s' % (v.ljust(varlen), doctag))
Example #10
0
def main():
    docfiles = []
    metadirs = []
    bbvars = set()
    undocumented = []
    docconf = ""
    onlydoctags = False

    # Collect and validate input
    try:
        opts, args = getopt.getopt(sys.argv[1:], "d:hm:t:T", ["help"])
    except getopt.GetoptError as err:
        print('%s' % str(err))
        usage()
        sys.exit(2)

    for o, a in opts:
        if o in ('-h', '--help'):
            usage()
            sys.exit(0)
        elif o == '-d':
            if os.path.isfile(a):
                docfiles.append(a)
            else:
                print('ERROR: documentation file %s is not a regular file' % a)
                sys.exit(3)
        elif o == '-m':
            if os.path.isdir(a):
                metadirs.append(a)
            else:
                print('ERROR: meta directory %s is not a directory' % a)
                sys.exit(4)
        elif o == "-t":
            if os.path.isfile(a):
                docconf = a
        elif o == "-T":
            onlydoctags = True
        else:
            assert False, "unhandled option"

    if len(docfiles) == 0:
        print('ERROR: no docfile specified')
        usage()
        sys.exit(5)

    if len(metadirs) == 0:
        print('ERROR: no metadir specified')
        usage()
        sys.exit(6)

    if onlydoctags and docconf == "":
        print('ERROR: no docconf specified')
        usage()
        sys.exit(7)

    prog = re.compile("^[^a-z]*$")
    with bb.tinfoil.Tinfoil() as tinfoil:
        tinfoil.prepare(config_only=False)
        parser = bb.codeparser.PythonParser('parser', None)
        datastore = tinfoil.config_data

        def bbvars_update(data):
            if prog.match(data):
                bbvars.add(data)
            if tinfoil.config_data.getVarFlag(data, 'python'):
                try:
                    parser.parse_python(tinfoil.config_data.getVar(data))
                except bb.data_smart.ExpansionError:
                    pass
                for var in parser.references:
                    if prog.match(var):
                        bbvars.add(var)
            else:
                try:
                    expandedVar = datastore.expandWithRefs(
                        datastore.getVar(data, False), data)
                    for var in expandedVar.references:
                        if prog.match(var):
                            bbvars.add(var)
                except bb.data_smart.ExpansionError:
                    pass

        # Use tinfoil to collect all the variable names globally
        for data in datastore:
            bbvars_update(data)

        # Collect variables from all recipes
        for recipe in tinfoil.all_recipe_files():
            print("Checking %s" % recipe)
            for data in tinfoil.parse_recipe_file(recipe):
                bbvars_update(data)

    documented_vars = collect_documented_vars(docfiles)

    # Check each var for documentation
    varlen = 0
    for v in bbvars:
        if len(v) > varlen:
            varlen = len(v)
        if not bbvar_is_documented(v, documented_vars):
            undocumented.append(v)
    undocumented.sort()
    varlen = varlen + 1

    # Report all undocumented variables
    print('Found %d undocumented bb variables (out of %d):' %
          (len(undocumented), len(bbvars)))
    header = '%s%s' % (str("VARIABLE").ljust(varlen), str("DOCTAG").ljust(7))
    print(header)
    print(str("").ljust(len(header), '='))
    for v in undocumented:
        doctag = bbvar_doctag(v, docconf)
        if not onlydoctags or not doctag == "":
            print('%s%s' % (v.ljust(varlen), doctag))
Example #11
0
def print_deps(tinfoil, abcd_file, rn):
    try:
        info = tinfoil.get_recipe_info(rn)
    except Exception:
        print('Failed to get recipe info for: %s' % rn)
        return []
    if not info:
        print('No recipe info found for: %s' % rn)
        return []
    append_files = tinfoil.get_file_appends(info.fn)
    appends = True
    data = tinfoil.parse_recipe_file(info.fn, appends, append_files)
    src_uri = data.getVar('SRC_URI').split()
    lic = data.getVar('LICENSE')
    summary = data.getVar('SUMMARY')
    description = data.getVar('DESCRIPTION')
    homepage = data.getVar('HOMEPAGE')
    srcrev = data.getVar('SRCREV')
    branch = data.getVar('BRANCH')
    depends = data.getVar('DEPENDS').split()

    abcd_file.write('- id:\n')
    abcd_file.write('    package_manager: "Yocto"\n')
    abcd_file.write('    name: "%s"\n' % info.pn)
    abcd_file.write('    version: "%s"\n' % info.pv)
    abcd_file.write('  declared_lics:\n')
    abcd_file.write('  - "%s"\n' % lic)
    if summary:
        abcd_file.write('  description: "%s"\n' % summary)
    else:
        abcd_file.write('  description: "%s"\n' % description)
    abcd_file.write('  homepage_url: "%s"\n' % homepage)
    abcd_file.write('  source_artifact:\n')
    repos = []
    for src in src_uri:
        # Strip options.
        # TODO: ignore files with apply=false?
        src = src.split(';', maxsplit=1)[0]
        src_type = src.split('://', maxsplit=1)[0]
        if src_type == 'file':
            # TODO: Get full path of patches and other files within the source
            # repo, not just the filesystem?
            fetch = bb.fetch2.Fetch([], data)
            local = fetch.localpath(src)
            abcd_file.write('  - "%s"\n' % local)
        else:
            abcd_file.write('  - "%s"\n' % src)
            if src_type != 'http' and src_type != 'https' and src_type != 'ftp' and src_type != 'ssh':
                repos.append(src)
    if len(repos) > 1:
        print('Multiple repos not fully supported yet. Pacakge: %s' % info.pn)
    for repo in repos:
        vcs_type, url = repo.split('://', maxsplit=1)
        abcd_file.write('  vcs:\n')
        if vcs_type == 'gitsm':
            vcs_type = 'git'
        abcd_file.write('    type: "%s"\n' % vcs_type)
        abcd_file.write('    url: "%s"\n' % url)
        # TODO: Actually support multiple repos here:
        abcd_file.write('    revision: "%s"\n' % srcrev)
        abcd_file.write('    branch: "%s"\n' % branch)

    abcd_file.write('  dependencies:\n')
    for dep in depends:
        abcd_file.write('  - "%s"\n' % dep)
        # TODO: search for transitive dependencies here? Each dependency will
        # get checked for its own dependencies sooner or later.

    return depends