Beispiel #1
0
def main():
    if '--help' in sys.argv:
        usage()
        sys.exit(0)
    if len(sys.argv) < 3:
        usage()
        sys.exit(1)

    utils.setup_django()
    import settings

    branch = utils.get_branch('master')
    fetchdir = settings.LAYER_FETCH_DIR

    import layerindex.models import LayerItem
    bitbakeitem = LayerItem()
    bitbakeitem.vcs_url = settings.BITBAKE_REPO_URL
    bitbakepath = os.path.join(fetchdir, bitbakeitem.get_fetch_dir())
    if getattr(settings, 'BITBAKE_PATH', ''):
        bitbakepath = os.path.join(bitbakepath, settings.BITBAKE_PATH)

    if not os.path.exists(bitbakepath):
        sys.stderr.write("Unable to find bitbake checkout at %s" % bitbakepath)
        sys.exit(1)

    lockfn = os.path.join(fetchdir, "layerindex.lock")
    lockfile = utils.lock_file(lockfn)
    if not lockfile:
        sys.stderr.write("Layer index lock timeout expired\n")
        sys.exit(1)
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, True)
        try:
            changeset = get_changeset(sys.argv[1])
            if not changeset:
                sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1])
                sys.exit(1)

            utils.setup_core_layer_sys_path(settings, branch.name)

            outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
        finally:
            tinfoil.shutdown()
    finally:
        utils.unlock_file(lockfile)

    if outp:
        print(outp)
    else:
        sys.stderr.write("No changes to write\n")
        sys.exit(1)

    shutil.rmtree(tempdir)
    sys.exit(0)
Beispiel #2
0
def main():
    if '--help' in sys.argv:
        usage()
        sys.exit(0)
    if len(sys.argv) < 3:
        usage()
        sys.exit(1)

    utils.setup_django()
    import settings

    branch = utils.get_branch('master')
    fetchdir = settings.LAYER_FETCH_DIR
    bitbakepath = os.path.join(fetchdir, 'bitbake')

    lockfn = os.path.join(fetchdir, "layerindex.lock")
    lockfile = utils.lock_file(lockfn)
    if not lockfile:
        sys.stderr.write("Layer index lock timeout expired\n")
        sys.exit(1)
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings, branch,
                                                     bitbakepath, True)

        changeset = get_changeset(sys.argv[1])
        if not changeset:
            sys.stderr.write("Unable to find changeset with id %s\n" %
                             sys.argv[1])
            sys.exit(1)

        outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
    finally:
        tinfoil.shutdown()
        utils.unlock_file(lockfile)

    if outp:
        print(outp)
    else:
        sys.stderr.write("No changes to write\n")
        sys.exit(1)

    shutil.rmtree(tempdir)
    sys.exit(0)
Beispiel #3
0
def main():
    if '--help' in sys.argv:
        usage()
        sys.exit(0)
    if len(sys.argv) < 3:
        usage()
        sys.exit(1)

    utils.setup_django()
    import settings

    branch = utils.get_branch('master')
    fetchdir = settings.LAYER_FETCH_DIR
    bitbakepath = os.path.join(fetchdir, 'bitbake')

    lockfn = os.path.join(fetchdir, "layerindex.lock")
    lockfile = utils.lock_file(lockfn)
    if not lockfile:
        sys.stderr.write("Layer index lock timeout expired\n")
        sys.exit(1)
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, True)

        changeset = get_changeset(sys.argv[1])
        if not changeset:
            sys.stderr.write("Unable to find changeset with id %s\n" % sys.argv[1])
            sys.exit(1)

        outp = generate_patches(tinfoil, fetchdir, changeset, sys.argv[2])
    finally:
        utils.unlock_file(lockfile)

    if outp:
        print outp
    else:
        sys.stderr.write("No changes to write\n")
        sys.exit(1)

    shutil.rmtree(tempdir)
    sys.exit(0)
Beispiel #4
0
def main():
    if LooseVersion(git.__version__) < '0.3.1':
        logger.error(
            "Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script"
        )
        sys.exit(1)

    parser = optparse.OptionParser(usage="""
    %prog [options]""")

    parser.add_option("-b",
                      "--branch",
                      help="Specify branch to update",
                      action="store",
                      dest="branch",
                      default='master')
    parser.add_option("-l",
                      "--layer",
                      help="Layer to update",
                      action="store",
                      dest="layer")
    parser.add_option(
        "-r",
        "--reload",
        help="Reload recipe data instead of updating since last update",
        action="store_true",
        dest="reload")
    parser.add_option(
        "",
        "--fullreload",
        help="Discard existing recipe data and fetch it from scratch",
        action="store_true",
        dest="fullreload")
    parser.add_option("-n",
                      "--dry-run",
                      help="Don't write any data back to the database",
                      action="store_true",
                      dest="dryrun")
    parser.add_option("",
                      "--nocheckout",
                      help="Don't check out branches",
                      action="store_true",
                      dest="nocheckout")
    parser.add_option("",
                      "--stop-on-error",
                      help="Stop on first parsing error",
                      action="store_true",
                      default=False,
                      dest="stop_on_error")
    parser.add_option("-i",
                      "--initial",
                      help="Print initial values parsed from layer.conf only",
                      action="store_true")
    parser.add_option("-d",
                      "--debug",
                      help="Enable debug output",
                      action="store_const",
                      const=logging.DEBUG,
                      dest="loglevel",
                      default=logging.INFO)
    parser.add_option("-q",
                      "--quiet",
                      help="Hide all output except error messages",
                      action="store_const",
                      const=logging.ERROR,
                      dest="loglevel")
    parser.add_option(
        "",
        "--keep-temp",
        help="Preserve temporary directory at the end instead of deleting it",
        action="store_true")

    options, args = parser.parse_args(sys.argv)
    if len(args) > 1:
        logger.error('unexpected argument "%s"' % args[1])
        parser.print_help()
        sys.exit(1)

    if options.fullreload:
        options.reload = True

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile
    from django.db import transaction

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    bitbakepath = os.path.join(fetchdir, 'bitbake')

    layer = utils.get_layer(options.layer)
    urldir = layer.get_fetch_dir()
    repodir = os.path.join(fetchdir, urldir)

    layerbranch = layer.get_layerbranch(options.branch)

    branchname = options.branch
    branchdesc = options.branch
    if layerbranch:
        if layerbranch.actual_branch:
            branchname = layerbranch.actual_branch
            branchdesc = "%s (%s)" % (options.branch, branchname)

    # Collect repo info
    repo = git.Repo(repodir)
    if repo.bare:
        logger.error('Repository %s is bare, not supported' % repodir)
        sys.exit(1)
    topcommit = repo.commit('origin/%s' % branchname)
    if options.nocheckout:
        topcommit = repo.commit('HEAD')

    tinfoil = None
    tempdir = None
    try:
        with transaction.atomic():
            newbranch = False
            if not layerbranch:
                # LayerBranch doesn't exist for this branch, create it
                newbranch = True
                layerbranch = LayerBranch()
                layerbranch.layer = layer
                layerbranch.branch = branch
                layerbranch_source = layer.get_layerbranch(branch)
                if not layerbranch_source:
                    layerbranch_source = layer.get_layerbranch(None)
                if layerbranch_source:
                    layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
                layerbranch.save()
                if layerbranch_source:
                    for maintainer in layerbranch_source.layermaintainer_set.all(
                    ):
                        maintainer.pk = None
                        maintainer.id = None
                        maintainer.layerbranch = layerbranch
                        maintainer.save()

            if layerbranch.vcs_subdir and not options.nocheckout:
                # Find latest commit in subdirectory
                # A bit odd to do it this way but apparently there's no other way in the GitPython API
                topcommit = next(
                    repo.iter_commits('origin/%s' % branchname,
                                      paths=layerbranch.vcs_subdir), None)

            layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
            layerdir_start = os.path.normpath(layerdir) + os.sep

            layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
            layermachines = Machine.objects.filter(layerbranch=layerbranch)
            layerdistros = Distro.objects.filter(layerbranch=layerbranch)
            layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
            layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
            layerincfiles = IncFile.objects.filter(layerbranch=layerbranch)
            if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial:
                # Check out appropriate branch
                if not options.nocheckout:
                    utils.checkout_layer_branch(layerbranch,
                                                repodir,
                                                logger=logger)

                logger.info("Collecting data for layer %s on branch %s" %
                            (layer.name, branchdesc))
                try:
                    (tinfoil, tempdir) = recipeparse.init_parser(
                        settings,
                        branch,
                        bitbakepath,
                        nocheckout=options.nocheckout,
                        logger=logger)
                except recipeparse.RecipeParseError as e:
                    logger.error(str(e))
                    sys.exit(1)
                logger.debug('Using temp directory %s' % tempdir)
                # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
                tinfoil.config_data.setVar('SUMMARY', '')
                # Clear the default value of DESCRIPTION so that we can see where it's not set
                tinfoil.config_data.setVar('DESCRIPTION', '')
                # Clear the default value of HOMEPAGE ('unknown')
                tinfoil.config_data.setVar('HOMEPAGE', '')
                # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
                # why won't they just fix that?!)
                tinfoil.config_data.setVar('LICENSE', '')

                layerconfparser = layerconfparse.LayerConfParse(
                    logger=logger, tinfoil=tinfoil)
                layer_config_data = layerconfparser.parse_layer(layerdir)
                if not layer_config_data:
                    logger.info(
                        "Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues"
                        % (layer.name, branchdesc))
                    layerconfparser.shutdown()
                    sys.exit(1)
                utils.set_layerbranch_collection_version(layerbranch,
                                                         layer_config_data,
                                                         logger=logger)
                if options.initial:
                    # Use print() rather than logger.info() since "-q" makes it print nothing.
                    for i in [
                            "BBFILE_COLLECTIONS", "LAYERVERSION",
                            "LAYERDEPENDS", "LAYERRECOMMENDS"
                    ]:
                        print(
                            '%s = "%s"' % (i,
                                           utils.get_layer_var(
                                               layer_config_data, i, logger)))
                    sys.exit(0)

                # Set up for recording patch info
                utils.setup_core_layer_sys_path(settings, branch.name)
                skip_patches = False
                try:
                    import oe.recipeutils
                except ImportError:
                    logger.warn(
                        'Failed to find lib/oe/recipeutils.py in layers - patch information will not be collected'
                    )
                    skip_patches = True

                utils.add_dependencies(layerbranch,
                                       layer_config_data,
                                       logger=logger)
                utils.add_recommends(layerbranch,
                                     layer_config_data,
                                     logger=logger)
                layerbranch.save()

                try:
                    config_data_copy = recipeparse.setup_layer(
                        tinfoil.config_data, fetchdir, layerdir, layer,
                        layerbranch, logger)
                except recipeparse.RecipeParseError as e:
                    logger.error(str(e))
                    sys.exit(1)

                if layerbranch.vcs_last_rev and not options.reload:
                    try:
                        diff = repo.commit(
                            layerbranch.vcs_last_rev).diff(topcommit)
                    except Exception as e:
                        logger.warn(
                            "Unable to get diff from last commit hash for layer %s - falling back to slow update: %s"
                            % (layer.name, str(e)))
                        diff = None
                else:
                    diff = None

                # We handle recipes specially to try to preserve the same id
                # when recipe upgrades happen (so that if a user bookmarks a
                # recipe page it remains valid)
                layerrecipes_delete = []
                layerrecipes_add = []

                # Check if any paths should be ignored because there are layers within this layer
                removedirs = []
                for root, dirs, files in os.walk(layerdir):
                    for diritem in dirs:
                        if os.path.exists(
                                os.path.join(root, diritem, 'conf',
                                             'layer.conf')):
                            removedirs.append(
                                os.path.join(root, diritem) + os.sep)

                if diff:
                    # Apply git changes to existing recipe list

                    if layerbranch.vcs_subdir:
                        subdir_start = os.path.normpath(
                            layerbranch.vcs_subdir) + os.sep
                    else:
                        subdir_start = ""

                    updatedrecipes = set()
                    dirtyrecipes = set()
                    other_deletes = []
                    other_adds = []
                    for diffitem in diff.iter_change_type('R'):
                        oldpath = diffitem.a_blob.path
                        newpath = diffitem.b_blob.path
                        skip = False
                        for removedir in removedirs:
                            # FIXME what about files moved into removedirs?
                            if oldpath.startswith(removedir):
                                skip = True
                                break
                        if skip:
                            continue
                        if oldpath.startswith(subdir_start):
                            if not newpath.startswith(subdir_start):
                                logger.debug(
                                    "Treating rename of %s to %s as a delete since new path is outside layer"
                                    % (oldpath, newpath))
                                other_deletes.append(diffitem)
                                continue
                            (oldtypename, oldfilepath,
                             oldfilename) = recipeparse.detect_file_type(
                                 oldpath, subdir_start)
                            (newtypename, newfilepath,
                             newfilename) = recipeparse.detect_file_type(
                                 newpath, subdir_start)
                            if oldtypename != newtypename:
                                # This is most likely to be a .inc file renamed to a .bb - and since
                                # there may be another recipe deleted at the same time we probably want
                                # to consider that, so just treat it as a delete and an add
                                logger.debug(
                                    "Treating rename of %s to %s as a delete and add (since type changed)"
                                    % (oldpath, newpath))
                                other_deletes.append(diffitem)
                                other_adds.append(diffitem)
                            elif oldtypename == 'recipe':
                                results = layerrecipes.filter(
                                    filepath=oldfilepath).filter(
                                        filename=oldfilename)
                                if len(results):
                                    recipe = results[0]
                                    logger.debug("Rename recipe %s to %s" %
                                                 (recipe, newpath))
                                    recipe.filepath = newfilepath
                                    recipe.filename = newfilename
                                    recipe.save()
                                    update_recipe_file(
                                        tinfoil, config_data_copy,
                                        os.path.join(layerdir, newfilepath),
                                        recipe, layerdir_start, repodir,
                                        options.stop_on_error, skip_patches)
                                    updatedrecipes.add(
                                        os.path.join(oldfilepath, oldfilename))
                                    updatedrecipes.add(
                                        os.path.join(newfilepath, newfilename))
                                else:
                                    logger.warn(
                                        "Renamed recipe %s could not be found"
                                        % oldpath)
                                    other_adds.append(diffitem)
                            elif oldtypename == 'bbappend':
                                results = layerappends.filter(
                                    filepath=oldfilepath).filter(
                                        filename=oldfilename)
                                if len(results):
                                    logger.debug(
                                        "Rename bbappend %s to %s" %
                                        (results[0],
                                         os.path.join(newfilepath,
                                                      newfilename)))
                                    results[0].filepath = newfilepath
                                    results[0].filename = newfilename
                                    results[0].save()
                                else:
                                    logger.warn(
                                        "Renamed bbappend %s could not be found"
                                        % oldpath)
                                    other_adds.append(diffitem)
                            elif oldtypename == 'machine':
                                results = layermachines.filter(
                                    name=oldfilename)
                                if len(results):
                                    logger.debug("Rename machine %s to %s" %
                                                 (results[0], newfilename))
                                    results[0].name = newfilename
                                    results[0].save()
                                else:
                                    logger.warn(
                                        "Renamed machine %s could not be found"
                                        % oldpath)
                                    other_adds.append(diffitem)
                            elif oldtypename == 'distro':
                                results = layerdistros.filter(name=oldfilename)
                                if len(results):
                                    logger.debug("Rename distro %s to %s" %
                                                 (results[0], newfilename))
                                    results[0].name = newfilename
                                    results[0].save()
                                else:
                                    logger.warn(
                                        "Renamed distro %s could not be found"
                                        % oldpath)
                                    other_adds.append(diffitem)
                            elif oldtypename == 'bbclass':
                                results = layerclasses.filter(name=oldfilename)
                                if len(results):
                                    logger.debug("Rename class %s to %s" %
                                                 (results[0], newfilename))
                                    results[0].name = newfilename
                                    results[0].save()
                                else:
                                    logger.warn(
                                        "Renamed class %s could not be found" %
                                        oldpath)
                                    other_adds.append(diffitem)
                            elif oldtypename == 'incfile':
                                results = layerincfiles.filter(
                                    path=os.path.join(oldfilepath,
                                                      oldfilename))
                                if len(results):
                                    logger.debug("Rename inc file %s to %s" %
                                                 (results[0], newfilename))
                                    results[0].name = newfilename
                                    results[0].save()
                                else:
                                    logger.warn(
                                        "Renamed inc file %s could not be found"
                                        % oldpath)
                                    other_adds.append(diffitem)

                            deps = RecipeFileDependency.objects.filter(
                                layerbranch=layerbranch).filter(path=oldpath)
                            for dep in deps:
                                dirtyrecipes.add(dep.recipe)

                    for diffitem in itertools.chain(diff.iter_change_type('D'),
                                                    other_deletes):
                        path = diffitem.a_blob.path
                        if path.startswith(subdir_start):
                            skip = False
                            for removedir in removedirs:
                                if path.startswith(removedir):
                                    skip = True
                                    break
                            if skip:
                                continue
                            (typename, filepath,
                             filename) = recipeparse.detect_file_type(
                                 path, subdir_start)
                            if typename == 'recipe':
                                values = layerrecipes.filter(
                                    filepath=filepath).filter(
                                        filename=filename).values(
                                            'id', 'filepath', 'filename', 'pn')
                                if len(values):
                                    layerrecipes_delete.append(values[0])
                                    logger.debug("Mark %s for deletion" %
                                                 values[0])
                                    updatedrecipes.add(
                                        os.path.join(values[0]['filepath'],
                                                     values[0]['filename']))
                                else:
                                    logger.warn(
                                        "Deleted recipe %s could not be found"
                                        % path)
                            elif typename == 'bbappend':
                                layerappends.filter(filepath=filepath).filter(
                                    filename=filename).delete()
                            elif typename == 'machine':
                                layermachines.filter(name=filename).delete()
                            elif typename == 'distro':
                                layerdistros.filter(name=filename).delete()
                            elif typename == 'bbclass':
                                layerclasses.filter(name=filename).delete()
                            elif typename == 'incfile':
                                layerincfiles.filter(path=os.path.join(
                                    filepath, filename)).delete()

                    for diffitem in itertools.chain(diff.iter_change_type('A'),
                                                    other_adds):
                        path = diffitem.b_blob.path
                        if path.startswith(subdir_start):
                            skip = False
                            for removedir in removedirs:
                                if path.startswith(removedir):
                                    skip = True
                                    break
                            if skip:
                                continue
                            (typename, filepath,
                             filename) = recipeparse.detect_file_type(
                                 path, subdir_start)
                            if typename == 'recipe':
                                layerrecipes_add.append(
                                    os.path.join(repodir, path))
                                logger.debug("Mark %s for addition" % path)
                                updatedrecipes.add(
                                    os.path.join(filepath, filename))
                            elif typename == 'bbappend':
                                append = BBAppend()
                                append.layerbranch = layerbranch
                                append.filename = filename
                                append.filepath = filepath
                                append.save()
                            elif typename == 'machine':
                                machine = Machine()
                                machine.layerbranch = layerbranch
                                machine.name = filename
                                update_machine_conf_file(
                                    os.path.join(repodir, path), machine)
                                machine.save()
                            elif typename == 'distro':
                                distro = Distro()
                                distro.layerbranch = layerbranch
                                distro.name = filename
                                update_distro_conf_file(
                                    os.path.join(repodir, path), distro,
                                    config_data_copy)
                                distro.save()
                            elif typename == 'bbclass':
                                bbclass = BBClass()
                                bbclass.layerbranch = layerbranch
                                bbclass.name = filename
                                bbclass.save()
                            elif typename == 'incfile':
                                incfile = IncFile()
                                incfile.layerbranch = layerbranch
                                incfile.path = os.path.join(filepath, filename)
                                incfile.save()

                    for diffitem in diff.iter_change_type('M'):
                        path = diffitem.b_blob.path
                        if path.startswith(subdir_start):
                            skip = False
                            for removedir in removedirs:
                                if path.startswith(removedir):
                                    skip = True
                                    break
                            if skip:
                                continue
                            (typename, filepath,
                             filename) = recipeparse.detect_file_type(
                                 path, subdir_start)
                            if typename == 'recipe':
                                logger.debug("Mark %s for update" % path)
                                results = layerrecipes.filter(
                                    filepath=filepath).filter(
                                        filename=filename)[:1]
                                if results:
                                    recipe = results[0]
                                    update_recipe_file(
                                        tinfoil, config_data_copy,
                                        os.path.join(layerdir, filepath),
                                        recipe, layerdir_start, repodir,
                                        options.stop_on_error, skip_patches)
                                    recipe.save()
                                    updatedrecipes.add(recipe.full_path())
                            elif typename == 'machine':
                                results = layermachines.filter(name=filename)
                                if results:
                                    machine = results[0]
                                    update_machine_conf_file(
                                        os.path.join(repodir, path), machine)
                                    machine.save()
                            elif typename == 'distro':
                                results = layerdistros.filter(name=filename)
                                if results:
                                    distro = results[0]
                                    update_distro_conf_file(
                                        os.path.join(repodir, path), distro,
                                        config_data_copy)
                                    distro.save()

                            deps = RecipeFileDependency.objects.filter(
                                layerbranch=layerbranch).filter(path=path)
                            for dep in deps:
                                dirtyrecipes.add(dep.recipe)

                    for recipe in dirtyrecipes:
                        if not recipe.full_path() in updatedrecipes:
                            update_recipe_file(
                                tinfoil, config_data_copy,
                                os.path.join(layerdir, recipe.filepath),
                                recipe, layerdir_start, repodir,
                                options.stop_on_error, skip_patches)
                else:
                    # Collect recipe data from scratch

                    layerrecipe_fns = []
                    if options.fullreload:
                        layerrecipes.delete()
                    else:
                        # First, check which recipes still exist
                        layerrecipe_values = layerrecipes.values(
                            'id', 'filepath', 'filename', 'pn')
                        for v in layerrecipe_values:
                            if v['filepath'].startswith('../'):
                                # FIXME: These recipes were present due to a bug (not handling renames
                                # to paths outside the layer) - this can be removed at some point in the future
                                preserve = False
                            else:
                                root = os.path.join(layerdir, v['filepath'])
                                fullpath = os.path.join(root, v['filename'])
                                if os.path.exists(fullpath):
                                    preserve = True
                                    for removedir in removedirs:
                                        if fullpath.startswith(removedir):
                                            preserve = False
                                            break
                                else:
                                    preserve = False

                            if preserve:
                                # Recipe still exists, update it
                                results = layerrecipes.filter(id=v['id'])[:1]
                                recipe = results[0]
                                update_recipe_file(tinfoil, config_data_copy,
                                                   root, recipe,
                                                   layerdir_start, repodir,
                                                   options.stop_on_error,
                                                   skip_patches)
                            else:
                                # Recipe no longer exists, mark it for later on
                                layerrecipes_delete.append(v)
                            layerrecipe_fns.append(fullpath)

                    layermachines.delete()
                    layerdistros.delete()
                    layerappends.delete()
                    layerclasses.delete()
                    for root, dirs, files in os.walk(layerdir):
                        if '.git' in dirs:
                            dirs.remove('.git')
                        for diritem in dirs[:]:
                            fullpath = os.path.join(root, diritem) + os.sep
                            if fullpath in removedirs:
                                dirs.remove(diritem)
                        for f in files:
                            fullpath = os.path.join(root, f)
                            (typename, _,
                             filename) = recipeparse.detect_file_type(
                                 fullpath, layerdir_start)
                            if typename == 'recipe':
                                if fullpath not in layerrecipe_fns:
                                    layerrecipes_add.append(fullpath)
                            elif typename == 'bbappend':
                                append = BBAppend()
                                append.layerbranch = layerbranch
                                append.filename = f
                                append.filepath = os.path.relpath(
                                    root, layerdir)
                                append.save()
                            elif typename == 'machine':
                                machine = Machine()
                                machine.layerbranch = layerbranch
                                machine.name = filename
                                update_machine_conf_file(fullpath, machine)
                                machine.save()
                            elif typename == 'distro':
                                distro = Distro()
                                distro.layerbranch = layerbranch
                                distro.name = filename
                                update_distro_conf_file(
                                    fullpath, distro, config_data_copy)
                                distro.save()
                            elif typename == 'bbclass':
                                bbclass = BBClass()
                                bbclass.layerbranch = layerbranch
                                bbclass.name = filename
                                bbclass.save()
                            elif typename == 'incfile':
                                incfile = IncFile()
                                incfile.layerbranch = layerbranch
                                incfile.path = os.path.relpath(
                                    fullpath, layerdir)
                                incfile.save()

                for added in layerrecipes_add:
                    # This is good enough without actually parsing the file
                    (pn, pv) = split_recipe_fn(added)
                    oldid = -1
                    for deleted in layerrecipes_delete:
                        if deleted['pn'] == pn:
                            oldid = deleted['id']
                            layerrecipes_delete.remove(deleted)
                            break
                    if oldid > -1:
                        # Reclaim a record we would have deleted
                        results = Recipe.objects.filter(id=oldid)[:1]
                        recipe = results[0]
                        logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
                    else:
                        # Create new record
                        logger.debug("Add new recipe %s" % added)
                        recipe = Recipe()
                    recipe.layerbranch = layerbranch
                    recipe.filename = os.path.basename(added)
                    root = os.path.dirname(added)
                    recipe.filepath = os.path.relpath(root, layerdir)
                    update_recipe_file(tinfoil, config_data_copy, root, recipe,
                                       layerdir_start, repodir,
                                       options.stop_on_error, skip_patches)
                    recipe.save()

                for deleted in layerrecipes_delete:
                    logger.debug("Delete %s" % deleted)
                    results = Recipe.objects.filter(id=deleted['id'])[:1]
                    recipe = results[0]
                    recipe.delete()

                # Save repo info
                layerbranch.vcs_last_rev = topcommit.hexsha
                layerbranch.vcs_last_commit = datetime.fromtimestamp(
                    topcommit.committed_date)
            else:
                logger.info("Layer %s is already up-to-date for branch %s" %
                            (layer.name, branchdesc))

            layerbranch.vcs_last_fetch = datetime.now()
            layerbranch.save()

            if options.dryrun:
                raise DryRunRollbackException()

    except KeyboardInterrupt:
        logger.warn("Update interrupted, changes to %s rolled back" %
                    layer.name)
        sys.exit(254)
    except SystemExit:
        raise
    except DryRunRollbackException:
        pass
    except:
        import traceback
        logger.error(traceback.format_exc().rstrip())
        sys.exit(1)
    finally:
        if tinfoil and (LooseVersion(bb.__version__) > LooseVersion("1.27")):
            tinfoil.shutdown()

    if tempdir:
        if options.keep_temp:
            logger.debug('Preserving temp directory %s' % tempdir)
        else:
            logger.debug('Deleting temp directory')
            utils.rmtree_force(tempdir)
    sys.exit(0)
Beispiel #5
0
def main():

    parser = optparse.OptionParser(usage="""
    %prog [options] <bitbakepath> <oeclassicpath>""")

    parser.add_option("-b",
                      "--branch",
                      help="Specify branch to import into",
                      action="store",
                      dest="branch",
                      default='oe-classic')
    parser.add_option("-l",
                      "--layer",
                      help="Specify layer to import into",
                      action="store",
                      dest="layer",
                      default='oe-classic')
    parser.add_option("-n",
                      "--dry-run",
                      help="Don't write any data back to the database",
                      action="store_true",
                      dest="dryrun")
    parser.add_option("-d",
                      "--debug",
                      help="Enable debug output",
                      action="store_const",
                      const=logging.DEBUG,
                      dest="loglevel",
                      default=logging.INFO)
    parser.add_option("-q",
                      "--quiet",
                      help="Hide all output except error messages",
                      action="store_const",
                      const=logging.ERROR,
                      dest="loglevel")

    options, args = parser.parse_args(sys.argv)
    if len(args) < 3:
        logger.error('You must specify bitbakepath and oeclassicpath')
        parser.print_help()
        sys.exit(1)
    if len(args) > 3:
        logger.error('unexpected argument "%s"' % args[3])
        parser.print_help()
        sys.exit(1)

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
    from django.db import transaction

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    res = list(LayerItem.objects.filter(name=options.layer)[:1])
    if res:
        layer = res[0]
    else:
        layer = LayerItem()
        layer.name = options.layer
        layer.status = 'P'
        layer.layer_type = 'M'
        layer.summary = 'OE-Classic'
        layer.description = 'OpenEmbedded-Classic'
        layer.vcs_url = 'git://git.openembedded.org/openembedded'
        layer.vcs_web_url = 'http://cgit.openembedded.org/openembedded'
        layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
        layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
        layer.comparison = True
        layer.save()

    layerbranch = layer.get_layerbranch(options.branch)
    if not layerbranch:
        # LayerBranch doesn't exist for this branch, create it
        layerbranch = LayerBranch()
        layerbranch.layer = layer
        layerbranch.branch = branch
        layerbranch.save()

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)
    fetchedrepos = []
    failedrepos = []

    bitbakepath = args[1]
    oeclassicpath = args[2]

    confparentdir = os.path.abspath(
        os.path.join(os.path.dirname(os.path.abspath(__file__)),
                     '../../oe-classic'))
    os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings,
                                                     branch,
                                                     bitbakepath,
                                                     nocheckout=True,
                                                     classic=True,
                                                     logger=logger)
    except recipeparse.RecipeParseError as e:
        logger.error(str(e))
        sys.exit(1)

    # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
    tinfoil.config_data.setVar('SUMMARY', '')
    # Clear the default value of DESCRIPTION so that we can see where it's not set
    tinfoil.config_data.setVar('DESCRIPTION', '')
    # Clear the default value of HOMEPAGE ('unknown')
    tinfoil.config_data.setVar('HOMEPAGE', '')

    try:
        with transaction.atomic():
            layerdir_start = os.path.normpath(oeclassicpath) + os.sep
            layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
            layermachines = Machine.objects.filter(layerbranch=layerbranch)
            layerdistros = Distro.objects.filter(layerbranch=layerbranch)
            layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
            layerclasses = BBClass.objects.filter(layerbranch=layerbranch)

            try:
                config_data_copy = recipeparse.setup_layer(
                    tinfoil.config_data, fetchdir, oeclassicpath, layer,
                    layerbranch, logger)
            except recipeparse.RecipeParseError as e:
                logger.error(str(e))
                sys.exit(1)

            layerrecipes.delete()
            layermachines.delete()
            layerdistros.delete()
            layerappends.delete()
            layerclasses.delete()
            for root, dirs, files in os.walk(oeclassicpath):
                if '.git' in dirs:
                    dirs.remove('.git')
                for f in files:
                    fullpath = os.path.join(root, f)
                    (typename, filepath,
                     filename) = recipeparse.detect_file_type(
                         fullpath, layerdir_start)
                    if typename == 'recipe':
                        recipe = ClassicRecipe()
                        recipe.layerbranch = layerbranch
                        recipe.filename = filename
                        recipe.filepath = filepath
                        update_recipe_file(tinfoil, config_data_copy, root,
                                           recipe, layerdir_start,
                                           oeclassicpath)
                        recipe.save()

            layerbranch.vcs_last_fetch = datetime.now()
            layerbranch.save()

            if options.dryrun:
                raise DryRunRollbackException()
    except DryRunRollbackException:
        pass
    except:
        import traceback
        traceback.print_exc()
    finally:
        tinfoil.shutdown()

    shutil.rmtree(tempdir)
    sys.exit(0)
def main():

    parser = optparse.OptionParser(
        usage = """
    %prog [options] <bitbakepath> <oeclassicpath>""")

    parser.add_option("-b", "--branch",
            help = "Specify branch to import into",
            action="store", dest="branch", default='oe-classic')
    parser.add_option("-l", "--layer",
            help = "Specify layer to import into",
            action="store", dest="layer", default='oe-classic')
    parser.add_option("-n", "--dry-run",
            help = "Don't write any data back to the database",
            action="store_true", dest="dryrun")
    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    parser.add_option("-q", "--quiet",
            help = "Hide all output except error messages",
            action="store_const", const=logging.ERROR, dest="loglevel")


    options, args = parser.parse_args(sys.argv)
    if len(args) < 3:
        logger.error('You must specify bitbakepath and oeclassicpath')
        parser.print_help()
        sys.exit(1)
    if len(args) > 3:
        logger.error('unexpected argument "%s"' % args[3])
        parser.print_help()
        sys.exit(1)

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
    from django.db import transaction

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    res = list(LayerItem.objects.filter(name=options.layer)[:1])
    if res:
        layer = res[0]
    else:
        layer = LayerItem()
        layer.name = options.layer
        layer.status = 'P'
        layer.layer_type = 'M'
        layer.summary = 'OE-Classic'
        layer.description = 'OpenEmbedded-Classic'
        layer.vcs_url =  'git://git.openembedded.org/openembedded'
        layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded'
        layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
        layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
        layer.classic = True
        layer.save()

    layerbranch = layer.get_layerbranch(options.branch)
    if not layerbranch:
        # LayerBranch doesn't exist for this branch, create it
        layerbranch = LayerBranch()
        layerbranch.layer = layer
        layerbranch.branch = branch
        layerbranch.save()

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)
    fetchedrepos = []
    failedrepos = []

    bitbakepath = args[1]
    oeclassicpath = args[2]

    confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic'))
    os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger)
    except recipeparse.RecipeParseError as e:
        logger.error(str(e))
        sys.exit(1)

    # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
    tinfoil.config_data.setVar('SUMMARY', '')
    # Clear the default value of DESCRIPTION so that we can see where it's not set
    tinfoil.config_data.setVar('DESCRIPTION', '')
    # Clear the default value of HOMEPAGE ('unknown')
    tinfoil.config_data.setVar('HOMEPAGE', '')

    transaction.enter_transaction_management()
    transaction.managed(True)
    try:
        layerdir_start = os.path.normpath(oeclassicpath) + os.sep
        layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
        layermachines = Machine.objects.filter(layerbranch=layerbranch)
        layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
        layerclasses = BBClass.objects.filter(layerbranch=layerbranch)

        try:
            config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
        except recipeparse.RecipeParseError as e:
            logger.error(str(e))
            transaction.rollback()
            sys.exit(1)

        layerrecipes.delete()
        layermachines.delete()
        layerappends.delete()
        layerclasses.delete()
        for root, dirs, files in os.walk(oeclassicpath):
            if '.git' in dirs:
                dirs.remove('.git')
            for f in files:
                fullpath = os.path.join(root, f)
                (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
                if typename == 'recipe':
                    recipe = ClassicRecipe()
                    recipe.layerbranch = layerbranch
                    recipe.filename = filename
                    recipe.filepath = filepath
                    update_recipe_file(config_data_copy, root, recipe, layerdir_start, oeclassicpath)
                    recipe.save()

        layerbranch.vcs_last_fetch = datetime.now()
        layerbranch.save()

        if options.dryrun:
            transaction.rollback()
        else:
            transaction.commit()
    except:
        import traceback
        traceback.print_exc()
        transaction.rollback()
    finally:
        transaction.leave_transaction_management()

    shutil.rmtree(tempdir)
    sys.exit(0)
Beispiel #7
0
def main():
    if LooseVersion(git.__version__) < '0.3.1':
        logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
        sys.exit(1)


    parser = optparse.OptionParser(
        usage = """
    %prog [options]""")

    parser.add_option("-b", "--branch",
            help = "Specify branch to update",
            action="store", dest="branch", default='master')
    parser.add_option("-l", "--layer",
            help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
            action="store", dest="layers")
    parser.add_option("-r", "--reload",
            help = "Reload recipe data instead of updating since last update",
            action="store_true", dest="reload")
    parser.add_option("", "--fullreload",
            help = "Discard existing recipe data and fetch it from scratch",
            action="store_true", dest="fullreload")
    parser.add_option("-n", "--dry-run",
            help = "Don't write any data back to the database",
            action="store_true", dest="dryrun")
    parser.add_option("-x", "--nofetch",
            help = "Don't fetch repositories",
            action="store_true", dest="nofetch")
    parser.add_option("", "--nocheckout",
            help = "Don't check out branches",
            action="store_true", dest="nocheckout")
    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    parser.add_option("-q", "--quiet",
            help = "Hide all output except error messages",
            action="store_const", const=logging.ERROR, dest="loglevel")

    options, args = parser.parse_args(sys.argv)
    if len(args) > 1:
        logger.error('unexpected argument "%s"' % args[1])
        parser.print_help()
        sys.exit(1)

    if options.fullreload:
        options.reload = True

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
    from django.db import transaction

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    if options.layers:
        layerquery = LayerItem.objects.filter(classic=False).filter(name__in=options.layers.split(','))
        if layerquery.count() == 0:
            logger.error('No layers matching specified query "%s"' % options.layers)
            sys.exit(1)
    else:
        layerquery = LayerItem.objects.filter(classic=False).filter(status='P')
        if layerquery.count() == 0:
            logger.info("No published layers to update")
            sys.exit(1)

    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)
    fetchedrepos = []
    failedrepos = []

    lockfn = os.path.join(fetchdir, "layerindex.lock")
    lockfile = utils.lock_file(lockfn)
    if not lockfile:
        logger.error("Layer index lock timeout expired")
        sys.exit(1)
    try:
        bitbakepath = os.path.join(fetchdir, 'bitbake')

        if not options.nofetch:
            # Fetch latest metadata from repositories
            for layer in layerquery:
                # Handle multiple layers in a single repo
                urldir = layer.get_fetch_dir()
                repodir = os.path.join(fetchdir, urldir)
                if not (layer.vcs_url in fetchedrepos or layer.vcs_url in failedrepos):
                    logger.info("Fetching remote repository %s" % layer.vcs_url)
                    out = None
                    try:
                        if not os.path.exists(repodir):
                            out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
                        else:
                            out = utils.runcmd("git fetch", repodir, logger=logger)
                    except Exception as e:
                        logger.error("Fetch of layer %s failed: %s" % (layer.name, str(e)))
                        failedrepos.append(layer.vcs_url)
                        continue
                    fetchedrepos.append(layer.vcs_url)

            if not fetchedrepos:
                logger.error("No repositories could be fetched, exiting")
                sys.exit(1)

            logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL)
            if not os.path.exists(bitbakepath):
                out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger)
            else:
                out = utils.runcmd("git fetch", bitbakepath, logger=logger)

        try:
            (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
        except recipeparse.RecipeParseError as e:
            logger.error(str(e))
            sys.exit(1)

        # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
        tinfoil.config_data.setVar('SUMMARY', '')
        # Clear the default value of DESCRIPTION so that we can see where it's not set
        tinfoil.config_data.setVar('DESCRIPTION', '')
        # Clear the default value of HOMEPAGE ('unknown')
        tinfoil.config_data.setVar('HOMEPAGE', '')
        # Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
        # why won't they just fix that?!)
        tinfoil.config_data.setVar('LICENSE', '')

        # Process and extract data from each layer
        for layer in layerquery:
            transaction.enter_transaction_management()
            transaction.managed(True)
            try:
                urldir = layer.get_fetch_dir()
                repodir = os.path.join(fetchdir, urldir)
                if layer.vcs_url in failedrepos:
                    logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
                    transaction.rollback()
                    continue

                layerbranch = layer.get_layerbranch(options.branch)

                branchname = options.branch
                branchdesc = options.branch
                if layerbranch:
                    if layerbranch.actual_branch:
                        branchname = layerbranch.actual_branch
                        branchdesc = "%s (%s)" % (options.branch, branchname)

                # Collect repo info
                repo = git.Repo(repodir)
                assert repo.bare == False
                try:
                    if options.nocheckout:
                        topcommit = repo.commit('HEAD')
                    else:
                        topcommit = repo.commit('origin/%s' % branchname)
                except:
                    if layerbranch:
                        logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
                    else:
                        logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
                    transaction.rollback()
                    continue

                newbranch = False
                if not layerbranch:
                    # LayerBranch doesn't exist for this branch, create it
                    newbranch = True
                    layerbranch = LayerBranch()
                    layerbranch.layer = layer
                    layerbranch.branch = branch
                    layerbranch_source = layer.get_layerbranch('master')
                    if not layerbranch_source:
                        layerbranch_source = layer.get_layerbranch(None)
                    if layerbranch_source:
                        layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
                    layerbranch.save()
                    if layerbranch_source:
                        for maintainer in layerbranch_source.layermaintainer_set.all():
                            maintainer.pk = None
                            maintainer.id = None
                            maintainer.layerbranch = layerbranch
                            maintainer.save()
                        for dep in layerbranch_source.dependencies_set.all():
                            dep.pk = None
                            dep.id = None
                            dep.layerbranch = layerbranch
                            dep.save()

                if layerbranch.vcs_subdir and not options.nocheckout:
                    # Find latest commit in subdirectory
                    # A bit odd to do it this way but apparently there's no other way in the GitPython API
                    topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
                    if not topcommit:
                        # This will error out if the directory is completely invalid or had never existed at this point
                        # If it previously existed but has since been deleted, you will get the revision where it was
                        # deleted - so we need to handle that case separately later
                        if newbranch:
                            logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
                        elif layerbranch.vcs_subdir:
                            logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
                        else:
                            logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
                        transaction.rollback()
                        continue

                layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
                layerdir_start = os.path.normpath(layerdir) + os.sep
                layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
                layermachines = Machine.objects.filter(layerbranch=layerbranch)
                layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
                layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
                if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
                    # Check out appropriate branch
                    if not options.nocheckout:
                        out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
                        out = utils.runcmd("git clean -f -x", repodir, logger=logger)

                    if layerbranch.vcs_subdir and not os.path.exists(layerdir):
                        if newbranch:
                            logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
                        else:
                            logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
                        transaction.rollback()
                        continue

                    if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
                        logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
                        transaction.rollback()
                        continue

                    logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))

                    try:
                        config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
                    except recipeparse.RecipeParseError as e:
                        logger.error(str(e))
                        transaction.rollback()
                        continue

                    if layerbranch.vcs_last_rev and not options.reload:
                        try:
                            diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
                        except Exception as e:
                            logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
                            diff = None
                    else:
                        diff = None

                    # We handle recipes specially to try to preserve the same id
                    # when recipe upgrades happen (so that if a user bookmarks a
                    # recipe page it remains valid)
                    layerrecipes_delete = []
                    layerrecipes_add = []

                    # Check if any paths should be ignored because there are layers within this layer
                    removedirs = []
                    for root, dirs, files in os.walk(layerdir):
                        for d in dirs:
                            if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
                                removedirs.append(os.path.join(root, d) + os.sep)

                    if diff:
                        # Apply git changes to existing recipe list

                        if layerbranch.vcs_subdir:
                            subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
                        else:
                            subdir_start = ""

                        updatedrecipes = set()
                        for d in diff.iter_change_type('D'):
                            path = d.a_blob.path
                            if path.startswith(subdir_start):
                                skip = False
                                for d in removedirs:
                                    if path.startswith(d):
                                        skip = True
                                        break
                                if skip:
                                    continue
                                (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
                                if typename == 'recipe':
                                    values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
                                    if len(values):
                                        layerrecipes_delete.append(values[0])
                                        logger.debug("Mark %s for deletion" % values[0])
                                        updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
                                    else:
                                        logger.warn("Deleted recipe %s could not be found" % path)
                                elif typename == 'bbappend':
                                    layerappends.filter(filepath=filepath).filter(filename=filename).delete()
                                elif typename == 'machine':
                                    layermachines.filter(name=filename).delete()
                                elif typename == 'bbclass':
                                    layerclasses.filter(name=filename).delete()

                        for d in diff.iter_change_type('A'):
                            path = d.b_blob.path
                            if path.startswith(subdir_start):
                                skip = False
                                for d in removedirs:
                                    if path.startswith(d):
                                        skip = True
                                        break
                                if skip:
                                    continue
                                (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
                                if typename == 'recipe':
                                    layerrecipes_add.append(os.path.join(repodir, path))
                                    logger.debug("Mark %s for addition" % path)
                                    updatedrecipes.add(os.path.join(filepath, filename))
                                elif typename == 'bbappend':
                                    append = BBAppend()
                                    append.layerbranch = layerbranch
                                    append.filename = filename
                                    append.filepath = filepath
                                    append.save()
                                elif typename == 'machine':
                                    machine = Machine()
                                    machine.layerbranch = layerbranch
                                    machine.name = filename
                                    update_machine_conf_file(os.path.join(repodir, path), machine)
                                    machine.save()
                                elif typename == 'bbclass':
                                    bbclass = BBClass()
                                    bbclass.layerbranch = layerbranch
                                    bbclass.name = filename
                                    bbclass.save()

                        dirtyrecipes = set()
                        for d in diff.iter_change_type('M'):
                            path = d.a_blob.path
                            if path.startswith(subdir_start):
                                skip = False
                                for d in removedirs:
                                    if path.startswith(d):
                                        skip = True
                                        break
                                if skip:
                                    continue
                                (typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
                                if typename == 'recipe':
                                    logger.debug("Mark %s for update" % path)
                                    results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
                                    if results:
                                        recipe = results[0]
                                        update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
                                        recipe.save()
                                        updatedrecipes.add(recipe.full_path())
                                elif typename == 'machine':
                                    results = layermachines.filter(name=filename)
                                    if results:
                                        machine = results[0]
                                        update_machine_conf_file(os.path.join(repodir, path), machine)
                                        machine.save()

                                deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
                                for dep in deps:
                                    dirtyrecipes.add(dep.recipe)

                        for recipe in dirtyrecipes:
                            if not recipe.full_path() in updatedrecipes:
                                update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
                    else:
                        # Collect recipe data from scratch

                        layerrecipe_fns = []
                        if options.fullreload:
                            layerrecipes.delete()
                        else:
                            # First, check which recipes still exist
                            layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
                            for v in layerrecipe_values:
                                root = os.path.join(layerdir, v['filepath'])
                                fullpath = os.path.join(root, v['filename'])
                                preserve = True
                                if os.path.exists(fullpath):
                                    for d in removedirs:
                                        if fullpath.startswith(d):
                                            preserve = False
                                            break
                                else:
                                    preserve = False

                                if preserve:
                                    # Recipe still exists, update it
                                    results = layerrecipes.filter(id=v['id'])[:1]
                                    recipe = results[0]
                                    update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
                                else:
                                    # Recipe no longer exists, mark it for later on
                                    layerrecipes_delete.append(v)
                                layerrecipe_fns.append(fullpath)

                        layermachines.delete()
                        layerappends.delete()
                        layerclasses.delete()
                        for root, dirs, files in os.walk(layerdir):
                            if '.git' in dirs:
                                dirs.remove('.git')
                            for d in dirs[:]:
                                fullpath = os.path.join(root, d) + os.sep
                                if fullpath in removedirs:
                                    dirs.remove(d)
                            for f in files:
                                fullpath = os.path.join(root, f)
                                (typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
                                if typename == 'recipe':
                                    if fullpath not in layerrecipe_fns:
                                        layerrecipes_add.append(fullpath)
                                elif typename == 'bbappend':
                                    append = BBAppend()
                                    append.layerbranch = layerbranch
                                    append.filename = f
                                    append.filepath = os.path.relpath(root, layerdir)
                                    append.save()
                                elif typename == 'machine':
                                    machine = Machine()
                                    machine.layerbranch = layerbranch
                                    machine.name = filename
                                    update_machine_conf_file(fullpath, machine)
                                    machine.save()
                                elif typename == 'bbclass':
                                    bbclass = BBClass()
                                    bbclass.layerbranch = layerbranch
                                    bbclass.name = filename
                                    bbclass.save()

                    for added in layerrecipes_add:
                        # This is good enough without actually parsing the file
                        (pn, pv) = split_recipe_fn(added)
                        oldid = -1
                        for deleted in layerrecipes_delete:
                            if deleted['pn'] == pn:
                                oldid = deleted['id']
                                layerrecipes_delete.remove(deleted)
                                break
                        if oldid > -1:
                            # Reclaim a record we would have deleted
                            results = Recipe.objects.filter(id=oldid)[:1]
                            recipe = results[0]
                            logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
                        else:
                            # Create new record
                            logger.debug("Add new recipe %s" % added)
                            recipe = Recipe()
                        recipe.layerbranch = layerbranch
                        recipe.filename = os.path.basename(added)
                        root = os.path.dirname(added)
                        recipe.filepath = os.path.relpath(root, layerdir)
                        update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
                        recipe.save()

                    for deleted in layerrecipes_delete:
                        logger.debug("Delete %s" % deleted)
                        results = Recipe.objects.filter(id=deleted['id'])[:1]
                        recipe = results[0]
                        recipe.delete()

                    # Save repo info
                    layerbranch.vcs_last_rev = topcommit.hexsha
                    layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
                else:
                    logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))

                layerbranch.vcs_last_fetch = datetime.now()
                layerbranch.save()

                if options.dryrun:
                    transaction.rollback()
                else:
                    transaction.commit()

                # Slightly hacky way of avoiding memory leaks
                bb.event.ui_queue = []
                bb.parse.parse_py.BBHandler.cached_statements = {}
                bb.codeparser.codeparsercache = bb.codeparser.CodeParserCache()
                if hasattr(bb.codeparser, 'codecache'):
                    bb.codeparser.codecache = bb.codeparser.SetCache()
                bb.fetch._checksum_cache = bb.checksum.FileChecksumCache()
                bb.fetch.urldata_cache = {}
                bb.fetch.saved_headrevs = {}
                bb.parse.__pkgsplit_cache__={}
                bb.parse.__mtime_cache = {}
                bb.parse.init_parser(tinfoil.config_data)

            except KeyboardInterrupt:
                transaction.rollback()
                logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
                break
            except:
                import traceback
                traceback.print_exc()
                transaction.rollback()
            finally:
                transaction.leave_transaction_management()

    finally:
        utils.unlock_file(lockfile)

    shutil.rmtree(tempdir)
    sys.exit(0)