def setLayers(self, bitbake, layers, targets):
        """ a word of attention: by convention, the first layer for any build will be poky! """

        assert self.be.sourcedir is not None

        layerlist = []
        nongitlayerlist = []

        # set layers in the layersource

        # 1. get a list of repos with branches, and map dirpaths for each layer
        gitrepos = {}

        # if we're using a remotely fetched version of bitbake add its git
        # details to the list of repos to clone
        if bitbake.giturl and bitbake.commit:
            gitrepos[(bitbake.giturl, bitbake.commit)] = []
            gitrepos[(bitbake.giturl, bitbake.commit)].append(
                ("bitbake", bitbake.dirpath))

        for layer in layers:
            # We don't need to git clone the layer for the CustomImageRecipe
            # as it's generated by us layer on if needed
            if CustomImageRecipe.LAYER_NAME in layer.name:
                continue

            # If we have local layers then we don't need clone them
            # For local layers giturl will be empty
            if not layer.giturl:
                nongitlayerlist.append(
                    layer.layer_version.layer.local_source_dir)
                continue

            if not (layer.giturl, layer.commit) in gitrepos:
                gitrepos[(layer.giturl, layer.commit)] = []
            gitrepos[(layer.giturl, layer.commit)].append(
                (layer.name, layer.dirpath))

        logger.debug("localhostbecontroller, our git repos are %s" %
                     pformat(gitrepos))

        # 2. Note for future use if the current source directory is a
        # checked-out git repos that could match a layer's vcs_url and therefore
        # be used to speed up cloning (rather than fetching it again).

        cached_layers = {}

        try:
            for remotes in self._shellcmd("git remote -v",
                                          self.be.sourcedir).split("\n"):
                try:
                    remote = remotes.split("\t")[1].split(" ")[0]
                    if remote not in cached_layers:
                        cached_layers[remote] = self.be.sourcedir
                except IndexError:
                    pass
        except ShellCmdException:
            # ignore any errors in collecting git remotes this is an optional
            # step
            pass

        logger.info("Using pre-checked out source for layer %s", cached_layers)

        # 3. checkout the repositories
        for giturl, commit in gitrepos.keys():
            localdirname = os.path.join(
                self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
            logger.debug(
                "localhostbecontroller: giturl %s:%s checking out in current directory %s"
                % (giturl, commit, localdirname))

            # see if our directory is a git repository
            if os.path.exists(localdirname):
                try:
                    localremotes = self._shellcmd("git remote -v",
                                                  localdirname)
                    if not giturl in localremotes and commit != 'HEAD':
                        raise BuildSetupException(
                            "Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
                            % (localdirname, ", ".join(
                                localremotes.split("\n")), giturl))
                except ShellCmdException:
                    # our localdirname might not be a git repository
                    #- that's fine
                    pass
            else:
                if giturl in cached_layers:
                    logger.debug("localhostbecontroller git-copying %s to %s" %
                                 (cached_layers[giturl], localdirname))
                    self._shellcmd("git clone \"%s\" \"%s\"" %
                                   (cached_layers[giturl], localdirname))
                    self._shellcmd("git remote remove origin", localdirname)
                    self._shellcmd("git remote add origin \"%s\"" % giturl,
                                   localdirname)
                else:
                    logger.debug("localhostbecontroller: cloning %s in %s" %
                                 (giturl, localdirname))
                    self._shellcmd('git clone "%s" "%s"' %
                                   (giturl, localdirname))

            # branch magic name "HEAD" will inhibit checkout
            if commit != "HEAD":
                logger.debug(
                    "localhostbecontroller: checking out commit %s to %s " %
                    (commit, localdirname))
                ref = commit if re.match('^[a-fA-F0-9]+$',
                                         commit) else 'origin/%s' % commit
                self._shellcmd(
                    'git fetch --all && git reset --hard "%s"' % ref,
                    localdirname)

            # take the localdirname as poky dir if we can find the oe-init-build-env
            if self.pokydirname is None and os.path.exists(
                    os.path.join(localdirname, "oe-init-build-env")):
                logger.debug(
                    "localhostbecontroller: selected poky dir name %s" %
                    localdirname)
                self.pokydirname = localdirname

                # make sure we have a working bitbake
                if not os.path.exists(os.path.join(self.pokydirname,
                                                   'bitbake')):
                    logger.debug(
                        "localhostbecontroller: checking bitbake into the poky dirname %s "
                        % self.pokydirname)
                    self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
                                   (bitbake.commit, bitbake.giturl,
                                    os.path.join(self.pokydirname, 'bitbake')))

            # verify our repositories
            for name, dirpath in gitrepos[(giturl, commit)]:
                localdirpath = os.path.join(localdirname, dirpath)
                logger.debug(
                    "localhostbecontroller: localdirpath expected '%s'" %
                    localdirpath)
                if not os.path.exists(localdirpath):
                    raise BuildSetupException(
                        "Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
                        % (localdirpath, giturl, commit))

                if name != "bitbake":
                    layerlist.append(localdirpath.rstrip("/"))

        logger.debug("localhostbecontroller: current layer list %s " %
                     pformat(layerlist))

        if self.pokydirname is None and os.path.exists(
                os.path.join(self.be.sourcedir, "oe-init-build-env")):
            logger.debug("localhostbecontroller: selected poky dir name %s" %
                         self.be.sourcedir)
            self.pokydirname = self.be.sourcedir

        # 5. create custom layer and add custom recipes to it
        for target in targets:
            try:
                customrecipe = CustomImageRecipe.objects.get(
                    name=target.target, project=bitbake.req.project)

                custom_layer_path = self.setup_custom_image_recipe(
                    customrecipe, layers)

                if os.path.isdir(custom_layer_path):
                    layerlist.append(custom_layer_path)

            except CustomImageRecipe.DoesNotExist:
                continue  # not a custom recipe, skip

        layerlist.extend(nongitlayerlist)
        logger.debug("\n\nset layers gives this list %s" % pformat(layerlist))
        self.islayerset = True
        return layerlist
    def triggerBuild(self, bitbake, layers, variables, targets, brbe):
        layers = self.setLayers(bitbake, layers, targets)

        # init build environment from the clone
        builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
        oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
        # init build environment
        try:
            custom_script = ToasterSetting.objects.get(
                name="CUSTOM_BUILD_INIT_SCRIPT").value
            custom_script = custom_script.replace("%BUILDDIR%", builddir)
            self._shellcmd("bash -c 'source %s'" % (custom_script))
        except ToasterSetting.DoesNotExist:
            self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
                           self.be.sourcedir)

        # update bblayers.conf
        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
        with open(bblconfpath, 'w') as bblayers:
            bblayers.write('# line added by toaster build control\n'
                           'BBLAYERS = "%s"' % ' '.join(layers))

        # write configuration file
        confpath = os.path.join(builddir, 'conf/toaster.conf')
        with open(confpath, 'w') as conf:
            for var in variables:
                conf.write('%s="%s"\n' % (var.name, var.value))
            conf.write('INHERIT+="toaster buildhistory"')

        # run bitbake server from the clone
        bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
        toasterlayers = os.path.join(builddir, "conf/toaster-bblayers.conf")
        self._shellcmd(
            'bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
            '--server-only -t xmlrpc -B 0.0.0.0:0\"' %
            (oe_init, builddir, bitbake, confpath, toasterlayers),
            self.be.sourcedir)

        # read port number from bitbake.lock
        self.be.bbport = ""
        bblock = os.path.join(builddir, 'bitbake.lock')
        with open(bblock) as fplock:
            for line in fplock:
                if ":" in line:
                    self.be.bbport = line.split(":")[-1].strip()
                    logger.debug("localhostbecontroller: bitbake port %s",
                                 self.be.bbport)
                    break

        if not self.be.bbport:
            raise BuildSetupException(
                "localhostbecontroller: can't read bitbake port from %s" %
                bblock)

        self.be.bbaddress = "localhost"
        self.be.bbstate = BuildEnvironment.SERVER_STARTED
        self.be.lock = BuildEnvironment.LOCK_RUNNING
        self.be.save()

        bbtargets = ''
        for target in targets:
            task = target.task
            if task:
                if not task.startswith('do_'):
                    task = 'do_' + task
                task = ':%s' % task
            bbtargets += '%s%s ' % (target.target, task)

        # run build with local bitbake. stop the server after the build.
        log = os.path.join(builddir, 'toaster_ui.log')
        local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                     'bitbake')
        self._shellcmd(['bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:-1" '
                        '%s %s -u toasterui --token="" >>%s 2>&1;'
                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:-1 %s -m)&\"' \
                        % (brbe, local_bitbake, bbtargets, log, bitbake)],
                        builddir, nowait=True)

        logger.debug('localhostbecontroller: Build launched, exiting. '
                     'Follow build logs at %s' % log)
예제 #3
0
    def triggerBuild(self, bitbake, layers, variables, targets, brbe):
        layers = self.setLayers(bitbake, layers, targets)
        is_merged_attr = bitbake.req.project.merged_attr

        git_env = os.environ.copy()
        # (note: add custom environment settings here)
        try:
            # insure that the project init/build uses the selected bitbake, and not Toaster's
            del git_env['TEMPLATECONF']
            del git_env['BBBASEDIR']
            del git_env['BUILDDIR']
        except KeyError:
            pass

        # init build environment from the clone
        if bitbake.req.project.builddir:
            builddir = bitbake.req.project.builddir
        else:
            builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
        oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
        # init build environment
        try:
            custom_script = ToasterSetting.objects.get(name="CUSTOM_BUILD_INIT_SCRIPT").value
            custom_script = custom_script.replace("%BUILDDIR%" ,builddir)
            self._shellcmd("bash -c 'source %s'" % (custom_script),env=git_env)
        except ToasterSetting.DoesNotExist:
            self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
                       self.be.sourcedir,env=git_env)

        # update bblayers.conf
        if not is_merged_attr:
            bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
            with open(bblconfpath, 'w') as bblayers:
                bblayers.write('# line added by toaster build control\n'
                               'BBLAYERS = "%s"' % ' '.join(layers))

            # write configuration file
            confpath = os.path.join(builddir, 'conf/toaster.conf')
            with open(confpath, 'w') as conf:
                for var in variables:
                    conf.write('%s="%s"\n' % (var.name, var.value))
                conf.write('INHERIT+="toaster buildhistory"')
        else:
            # Append the Toaster-specific values directly to the bblayers.conf
            bblconfpath = os.path.join(builddir, "conf/bblayers.conf")
            bblconfpath_save = os.path.join(builddir, "conf/bblayers.conf.save")
            shutil.copyfile(bblconfpath, bblconfpath_save)
            with open(bblconfpath) as bblayers:
                content = bblayers.readlines()
            do_write = True
            was_toaster = False
            with open(bblconfpath,'w') as bblayers:
                for line in content:
                    #line = line.strip('\n')
                    if 'TOASTER_CONFIG_PROLOG' in line:
                        do_write = False
                        was_toaster = True
                    elif 'TOASTER_CONFIG_EPILOG' in line:
                        do_write = True
                    elif do_write:
                        bblayers.write(line)
                if not was_toaster:
                    bblayers.write('\n')
                bblayers.write('#=== TOASTER_CONFIG_PROLOG ===\n')
                bblayers.write('BBLAYERS = "\\\n')
                for layer in layers:
                    bblayers.write('  %s \\\n' % layer)
                bblayers.write('  "\n')
                bblayers.write('#=== TOASTER_CONFIG_EPILOG ===\n')
            # Append the Toaster-specific values directly to the local.conf
            bbconfpath = os.path.join(builddir, "conf/local.conf")
            bbconfpath_save = os.path.join(builddir, "conf/local.conf.save")
            shutil.copyfile(bbconfpath, bbconfpath_save)
            with open(bbconfpath) as f:
                content = f.readlines()
            do_write = True
            was_toaster = False
            with open(bbconfpath,'w') as conf:
                for line in content:
                    #line = line.strip('\n')
                    if 'TOASTER_CONFIG_PROLOG' in line:
                        do_write = False
                        was_toaster = True
                    elif 'TOASTER_CONFIG_EPILOG' in line:
                        do_write = True
                    elif do_write:
                        conf.write(line)
                if not was_toaster:
                    conf.write('\n')
                conf.write('#=== TOASTER_CONFIG_PROLOG ===\n')
                for var in variables:
                    if (not var.name.startswith("INTERNAL_")) and (not var.name == "BBLAYERS"):
                        conf.write('%s="%s"\n' % (var.name, var.value))
                conf.write('#=== TOASTER_CONFIG_EPILOG ===\n')

        # If 'target' is just the project preparation target, then we are done
        for target in targets:
            if "_PROJECT_PREPARE_" == target.target:
                logger.debug('localhostbecontroller: Project has been prepared. Done.')
                # Update the Build Request and release the build environment
                bitbake.req.state = BuildRequest.REQ_COMPLETED
                bitbake.req.save()
                self.be.lock = BuildEnvironment.LOCK_FREE
                self.be.save()
                # Close the project build and progress bar
                bitbake.req.build.outcome = Build.SUCCEEDED
                bitbake.req.build.save()
                # Update the project status
                bitbake.req.project.set_variable(Project.PROJECT_SPECIFIC_STATUS,Project.PROJECT_SPECIFIC_CLONING_SUCCESS)
                signal_runbuilds()
                return

        # clean the Toaster to build environment
        env_clean = 'unset BBPATH;' # clean BBPATH for <= YP-2.4.0

        # run bitbake server from the clone if available
        # otherwise pick it from the PATH
        bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
        if not os.path.exists(bitbake):
            logger.info("Bitbake not available under %s, will try to use it from PATH" %
                        self.pokydirname)
            for path in os.environ["PATH"].split(os.pathsep):
                if os.path.exists(os.path.join(path, 'bitbake')):
                    bitbake = os.path.join(path, 'bitbake')
                    break
            else:
                logger.error("Looks like Bitbake is not available, please fix your environment")

        toasterlayers = os.path.join(builddir,"conf/toaster-bblayers.conf")
        if not is_merged_attr:
            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
                           builddir, bitbake, confpath, toasterlayers), self.be.sourcedir)
        else:
            self._shellcmd('%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s '
                           '--server-only -B 0.0.0.0:0\"' % (env_clean, oe_init,
                           builddir, bitbake), self.be.sourcedir)

        # read port number from bitbake.lock
        self.be.bbport = -1
        bblock = os.path.join(builddir, 'bitbake.lock')
        # allow 10 seconds for bb lock file to appear but also be populated
        for lock_check in range(10):
            if not os.path.exists(bblock):
                logger.debug("localhostbecontroller: waiting for bblock file to appear")
                time.sleep(1)
                continue
            if 10 < os.stat(bblock).st_size:
                break
            logger.debug("localhostbecontroller: waiting for bblock content to appear")
            time.sleep(1)
        else:
            raise BuildSetupException("Cannot find bitbake server lock file '%s'. Aborting." % bblock)

        with open(bblock) as fplock:
            for line in fplock:
                if ":" in line:
                    self.be.bbport = line.split(":")[-1].strip()
                    logger.debug("localhostbecontroller: bitbake port %s", self.be.bbport)
                    break

        if -1 == self.be.bbport:
            raise BuildSetupException("localhostbecontroller: can't read bitbake port from %s" % bblock)

        self.be.bbaddress = "localhost"
        self.be.bbstate = BuildEnvironment.SERVER_STARTED
        self.be.lock = BuildEnvironment.LOCK_RUNNING
        self.be.save()

        bbtargets = ''
        for target in targets:
            task = target.task
            if task:
                if not task.startswith('do_'):
                    task = 'do_' + task
                task = ':%s' % task
            bbtargets += '%s%s ' % (target.target, task)

        # run build with local bitbake. stop the server after the build.
        log = os.path.join(builddir, 'toaster_ui.log')
        local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                     'bitbake')
        if not is_merged_attr:
            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                        '%s %s -u toasterui  --read %s --read %s --token="" >>%s 2>&1;'
                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
                        self.be.bbport, bitbake,)],
                        builddir, nowait=True)
        else:
            self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                        '%s %s -u toasterui  --token="" >>%s 2>&1;'
                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, log,
                        self.be.bbport, bitbake,)],
                        builddir, nowait=True)

        logger.debug('localhostbecontroller: Build launched, exiting. '
                     'Follow build logs at %s' % log)
예제 #4
0
    def triggerBuild(self, bitbake, layers, variables, targets, brbe):
        layers = self.setLayers(bitbake, layers, targets)

        # init build environment from the clone
        builddir = '%s-toaster-%d' % (self.be.builddir, bitbake.req.project.id)
        oe_init = os.path.join(self.pokydirname, 'oe-init-build-env')
        # init build environment
        try:
            custom_script = ToasterSetting.objects.get(
                name="CUSTOM_BUILD_INIT_SCRIPT").value
            custom_script = custom_script.replace("%BUILDDIR%", builddir)
            self._shellcmd("bash -c 'source %s'" % (custom_script))
        except ToasterSetting.DoesNotExist:
            self._shellcmd("bash -c 'source %s %s'" % (oe_init, builddir),
                           self.be.sourcedir)

        # update bblayers.conf
        bblconfpath = os.path.join(builddir, "conf/toaster-bblayers.conf")
        with open(bblconfpath, 'w') as bblayers:
            bblayers.write('# line added by toaster build control\n'
                           'BBLAYERS = "%s"' % ' '.join(layers))

        # write configuration file
        confpath = os.path.join(builddir, 'conf/toaster.conf')
        with open(confpath, 'w') as conf:
            for var in variables:
                conf.write('%s="%s"\n' % (var.name, var.value))
            conf.write('INHERIT+="toaster buildhistory"')

        # clean the Toaster to build environment
        env_clean = 'unset BBPATH;'  # clean BBPATH for <= YP-2.4.0

        # run bitbake server from the clone if available
        # otherwise pick it from the PATH
        bitbake = os.path.join(self.pokydirname, 'bitbake', 'bin', 'bitbake')
        if not os.path.exists(bitbake):
            logger.info(
                "Bitbake not available under %s, will try to use it from PATH"
                % self.pokydirname)
            for path in os.environ["PATH"].split(os.pathsep):
                if os.path.exists(os.path.join(path, 'bitbake')):
                    bitbake = os.path.join(path, 'bitbake')
                    logger.info("Found Bitbake at: %s" % path)
                    break
            else:
                logger.error(
                    "Looks like Bitbake is not available, please fix your environment"
                )

        # run bitbake server from the clone
        toasterlayers = os.path.join(builddir, "conf/toaster-bblayers.conf")
        self._shellcmd(
            '%s bash -c \"source %s %s; BITBAKE_UI="knotty" %s --read %s --read %s '
            '--server-only -B 0.0.0.0:0\"' %
            (env_clean, oe_init, builddir, bitbake, confpath, toasterlayers),
            self.be.sourcedir)

        # read port number from bitbake.lock
        self.be.bbport = -1
        bblock = os.path.join(builddir, 'bitbake.lock')
        # allow 10 seconds for bb lock file to appear but also be populated
        for lock_check in range(10):
            if not os.path.exists(bblock):
                logger.debug(
                    "localhostbecontroller: waiting for bblock file to appear")
                time.sleep(1)
                continue
            if 10 < os.stat(bblock).st_size:
                break
            logger.debug(
                "localhostbecontroller: waiting for bblock content to appear")
            time.sleep(1)
        else:
            raise BuildSetupException(
                "Cannot find bitbake server lock file '%s'. Aborting." %
                bblock)

        with open(bblock) as fplock:
            for line in fplock:
                if ":" in line:
                    self.be.bbport = line.split(":")[-1].strip()
                    logger.debug("localhostbecontroller: bitbake port %s",
                                 self.be.bbport)
                    break

        if -1 == self.be.bbport:
            raise BuildSetupException(
                "localhostbecontroller: can't read bitbake port from %s" %
                bblock)

        self.be.bbaddress = "localhost"
        self.be.bbstate = BuildEnvironment.SERVER_STARTED
        self.be.lock = BuildEnvironment.LOCK_RUNNING
        self.be.save()

        bbtargets = ''
        for target in targets:
            task = target.task
            if task:
                if not task.startswith('do_'):
                    task = 'do_' + task
                task = ':%s' % task
            bbtargets += '%s%s ' % (target.target, task)

        # run build with local bitbake. stop the server after the build.
        log = os.path.join(builddir, 'toaster_ui.log')
        local_bitbake = os.path.join(os.path.dirname(os.getenv('BBBASEDIR')),
                                     'bitbake')
        self._shellcmd(['%s bash -c \"(TOASTER_BRBE="%s" BBSERVER="0.0.0.0:%s" '
                        '%s %s -u toasterui  --read %s --read %s --token="" >>%s 2>&1;'
                        'BITBAKE_UI="knotty" BBSERVER=0.0.0.0:%s %s -m)&\"' \
                        % (env_clean, brbe, self.be.bbport, local_bitbake, bbtargets, confpath, toasterlayers, log,
                        self.be.bbport, bitbake,)],
                        builddir, nowait=True)

        logger.debug('localhostbecontroller: Build launched, exiting. '
                     'Follow build logs at %s' % log)
예제 #5
0
    def setLayers(self, bitbake, layers, targets):
        """ a word of attention: by convention, the first layer for any build will be poky! """

        assert self.be.sourcedir is not None

        layerlist = []
        nongitlayerlist = []

        # set layers in the layersource

        # 1. get a list of repos with branches, and map dirpaths for each layer
        gitrepos = {}

        gitrepos[(bitbake.giturl, bitbake.commit)] = []
        gitrepos[(bitbake.giturl, bitbake.commit)].append(
            ("bitbake", bitbake.dirpath))

        for layer in layers:
            # We don't need to git clone the layer for the CustomImageRecipe
            # as it's generated by us layer on if needed
            if CustomImageRecipe.LAYER_NAME in layer.name:
                continue

            # If we have local layers then we don't need clone them
            # For local layers giturl will be empty
            if not layer.giturl:
                nongitlayerlist.append(
                    layer.layer_version.layer.local_source_dir)
                continue

            if not (layer.giturl, layer.commit) in gitrepos:
                gitrepos[(layer.giturl, layer.commit)] = []
            gitrepos[(layer.giturl, layer.commit)].append(
                (layer.name, layer.dirpath))

        logger.debug("localhostbecontroller, our git repos are %s" %
                     pformat(gitrepos))

        # 2. Note for future use if the current source directory is a
        # checked-out git repos that could match a layer's vcs_url and therefore
        # be used to speed up cloning (rather than fetching it again).

        cached_layers = {}

        try:
            for remotes in self._shellcmd("git remote -v",
                                          self.be.sourcedir).split("\n"):
                try:
                    remote = remotes.split("\t")[1].split(" ")[0]
                    if remote not in cached_layers:
                        cached_layers[remote] = self.be.sourcedir
                except IndexError:
                    pass
        except ShellCmdException:
            # ignore any errors in collecting git remotes this is an optional
            # step
            pass

        logger.info("Using pre-checked out source for layer %s", cached_layers)

        # 3. checkout the repositories
        for giturl, commit in gitrepos.keys():
            localdirname = os.path.join(
                self.be.sourcedir, self.getGitCloneDirectory(giturl, commit))
            logger.debug(
                "localhostbecontroller: giturl %s:%s checking out in current directory %s"
                % (giturl, commit, localdirname))

            # make sure our directory is a git repository
            if os.path.exists(localdirname):
                localremotes = self._shellcmd("git remote -v", localdirname)
                if not giturl in localremotes:
                    raise BuildSetupException(
                        "Existing git repository at %s, but with different remotes ('%s', expected '%s'). Toaster will not continue out of fear of damaging something."
                        % (localdirname, ", ".join(
                            localremotes.split("\n")), giturl))
            else:
                if giturl in cached_layers:
                    logger.debug("localhostbecontroller git-copying %s to %s" %
                                 (cached_layers[giturl], localdirname))
                    self._shellcmd("git clone \"%s\" \"%s\"" %
                                   (cached_layers[giturl], localdirname))
                    self._shellcmd("git remote remove origin", localdirname)
                    self._shellcmd("git remote add origin \"%s\"" % giturl,
                                   localdirname)
                else:
                    logger.debug("localhostbecontroller: cloning %s in %s" %
                                 (giturl, localdirname))
                    self._shellcmd('git clone "%s" "%s"' %
                                   (giturl, localdirname))

            # branch magic name "HEAD" will inhibit checkout
            if commit != "HEAD":
                logger.debug(
                    "localhostbecontroller: checking out commit %s to %s " %
                    (commit, localdirname))
                ref = commit if re.match('^[a-fA-F0-9]+$',
                                         commit) else 'origin/%s' % commit
                self._shellcmd(
                    'git fetch --all && git reset --hard "%s"' % ref,
                    localdirname)

            # take the localdirname as poky dir if we can find the oe-init-build-env
            if self.pokydirname is None and os.path.exists(
                    os.path.join(localdirname, "oe-init-build-env")):
                logger.debug(
                    "localhostbecontroller: selected poky dir name %s" %
                    localdirname)
                self.pokydirname = localdirname

                # make sure we have a working bitbake
                if not os.path.exists(os.path.join(self.pokydirname,
                                                   'bitbake')):
                    logger.debug(
                        "localhostbecontroller: checking bitbake into the poky dirname %s "
                        % self.pokydirname)
                    self._shellcmd("git clone -b \"%s\" \"%s\" \"%s\" " %
                                   (bitbake.commit, bitbake.giturl,
                                    os.path.join(self.pokydirname, 'bitbake')))

            # verify our repositories
            for name, dirpath in gitrepos[(giturl, commit)]:
                localdirpath = os.path.join(localdirname, dirpath)
                logger.debug(
                    "localhostbecontroller: localdirpath expected '%s'" %
                    localdirpath)
                if not os.path.exists(localdirpath):
                    raise BuildSetupException(
                        "Cannot find layer git path '%s' in checked out repository '%s:%s'. Aborting."
                        % (localdirpath, giturl, commit))

                if name != "bitbake":
                    layerlist.append(localdirpath.rstrip("/"))

        logger.debug("localhostbecontroller: current layer list %s " %
                     pformat(layerlist))

        # 5. create custom layer and add custom recipes to it
        layerpath = os.path.join(self.be.builddir,
                                 CustomImageRecipe.LAYER_NAME)
        for target in targets:
            try:
                customrecipe = CustomImageRecipe.objects.get(
                    name=target.target, project=bitbake.req.project)
            except CustomImageRecipe.DoesNotExist:
                continue  # not a custom recipe, skip

            # create directory structure
            for name in ("conf", "recipes"):
                path = os.path.join(layerpath, name)
                if not os.path.isdir(path):
                    os.makedirs(path)

            # create layer.oonf
            config = os.path.join(layerpath, "conf", "layer.conf")
            if not os.path.isfile(config):
                with open(config, "w") as conf:
                    conf.write(
                        'BBPATH .= ":${LAYERDIR}"\nBBFILES += "${LAYERDIR}/recipes/*.bb"\n'
                    )

            # Update the Layer_Version dirpath that has our base_recipe in
            # to be able to read the base recipe to then  generate the
            # custom recipe.
            br_layer_base_recipe = layers.get(
                layer_version=customrecipe.base_recipe.layer_version)

            br_layer_base_dirpath = \
                    os.path.join(self.be.sourcedir,
                                 self.getGitCloneDirectory(
                                     br_layer_base_recipe.giturl,
                                     br_layer_base_recipe.commit),
                                 customrecipe.base_recipe.layer_version.dirpath
                                )

            customrecipe.base_recipe.layer_version.dirpath = \
                         br_layer_base_dirpath

            customrecipe.base_recipe.layer_version.save()

            # create recipe
            recipe_path = \
                    os.path.join(layerpath, "recipes", "%s.bb" % target.target)
            with open(recipe_path, "w") as recipef:
                recipef.write(customrecipe.generate_recipe_file_contents())

            # Update the layer and recipe objects
            customrecipe.layer_version.dirpath = layerpath
            customrecipe.layer_version.save()

            customrecipe.file_path = recipe_path
            customrecipe.save()

            # create *Layer* objects needed for build machinery to work
            BRLayer.objects.get_or_create(req=target.req,
                                          name=layer.name,
                                          dirpath=layerpath,
                                          giturl="file://%s" % layerpath)
        if os.path.isdir(layerpath):
            layerlist.append(layerpath)

        self.islayerset = True
        layerlist.extend(nongitlayerlist)
        return layerlist