예제 #1
0
def main():

    parser = optparse.OptionParser(
        usage = """
    %prog [options]""")

    options, args = parser.parse_args(sys.argv)

    utils.setup_django()
    from layerindex.models import LayerItem, LayerBranch, LayerDependency
    from django.db import transaction

    import httplib
    conn = httplib.HTTPConnection("www.openembedded.org")
    conn.request("GET", "/wiki/LayerIndex?action=raw")
    resp = conn.getresponse()
    if resp.status in [200, 302]:
        data = resp.read()
        in_table = False
        layer_type = 'M'
        nowiki_re = re.compile(r'</?nowiki>')
        link_re = re.compile(r'\[(http.*) +link\]')
        readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
        master_branch = utils.get_branch('master')
        core_layer = None
        with transaction.atomic():
            for line in data.splitlines():
                if line.startswith('{|'):
                    in_table = True
                    continue
                if in_table:
                    if line.startswith('|}'):
                        # We're done
                        break
                    elif line.startswith('!'):
                        section = line.split('|', 1)[1].strip("'")
                        if section.startswith('Base'):
                            layer_type = 'A'
                        elif section.startswith('Board'):
                            layer_type = 'B'
                        elif section.startswith('Software'):
                            layer_type = 'S'
                        elif section.startswith('Distribution'):
                            layer_type = 'D'
                        else:
                            layer_type = 'M'
                    elif not line.startswith('|-'):
                        if line.startswith("|| ''"):
                            continue
                        fields = line.split('||')
                        layer = LayerItem()
                        layer.name = fields[1].strip()
                        if ' ' in layer.name:
                            logger.warn('Skipping layer %s - name invalid' % layer.name)
                            continue
                        logger.info('Adding layer %s' % layer.name)
                        layer.status = 'P'
                        layer.layer_type = layer_type
                        layer.summary = fields[2].strip()
                        layer.description = layer.summary
                        if len(fields) > 6:
                            res = link_re.match(fields[6].strip())
                            if res:
                                link = res.groups(1)[0].strip()
                                if link.endswith('/README') or readme_re.search(link):
                                    link = 'README'
                                layer.usage_url = link

                        repoval = nowiki_re.sub('', fields[4]).strip()
                        layer.vcs_url = repoval
                        if repoval.startswith('git://git.openembedded.org/'):
                            reponame = re.sub('^.*/', '', repoval)
                            layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame
                            layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
                            layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
                            layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%'
                        elif repoval.startswith('git://git.yoctoproject.org/'):
                            reponame = re.sub('^.*/', '', repoval)
                            layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
                            layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
                            layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
                            layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%'
                        elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'):
                            reponame = re.sub('^.*github.com/', '', repoval)
                            reponame = re.sub('.git$', '', reponame)
                            layer.vcs_web_url = 'http://github.com/' + reponame
                            layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
                            layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
                            layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%'
                        elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'):
                            reponame = re.sub('^.*gitlab.com/', '', repoval)
                            reponame = re.sub('.git$', '', reponame)
                            layer.vcs_web_url = 'http://gitlab.com/' + reponame
                            layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/'
                            layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/'
                            layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%'
                        elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'):
                            reponame = re.sub('^.*bitbucket.org/', '', repoval)
                            reponame = re.sub('.git$', '', reponame)
                            layer.vcs_web_url = 'http://bitbucket.org/' + reponame
                            layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
                            layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
                            layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%'
                        elif '.git' in repoval:
                            res = link_re.match(fields[5].strip())
                            layer.vcs_web_url = res.groups(1)[0]
                            layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
                            layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
                            layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url)

                        layer.save()
                        layerbranch = LayerBranch()
                        layerbranch.layer = layer
                        layerbranch.branch = master_branch
                        layerbranch.vcs_subdir = fields[3].strip()
                        layerbranch.save()
                        if layer.name != 'openembedded-core':
                            if not core_layer:
                                core_layer = utils.get_layer('openembedded-core')
                            if core_layer:
                                layerdep = LayerDependency()
                                layerdep.layerbranch = layerbranch
                                layerdep.dependency = core_layer
                                layerdep.save()
    else:
        logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))

    sys.exit(0)
예제 #2
0
def main():

    parser = optparse.OptionParser(
        usage = """
    %prog [options] <bitbakepath> <oeclassicpath>""")

    parser.add_option("-b", "--branch",
            help = "Specify branch to import into",
            action="store", dest="branch", default='oe-classic')
    parser.add_option("-l", "--layer",
            help = "Specify layer to import into",
            action="store", dest="layer", default='oe-classic')
    parser.add_option("-n", "--dry-run",
            help = "Don't write any data back to the database",
            action="store_true", dest="dryrun")
    parser.add_option("-d", "--debug",
            help = "Enable debug output",
            action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
    parser.add_option("-q", "--quiet",
            help = "Hide all output except error messages",
            action="store_const", const=logging.ERROR, dest="loglevel")


    options, args = parser.parse_args(sys.argv)
    if len(args) < 3:
        logger.error('You must specify bitbakepath and oeclassicpath')
        parser.print_help()
        sys.exit(1)
    if len(args) > 3:
        logger.error('unexpected argument "%s"' % args[3])
        parser.print_help()
        sys.exit(1)

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
    from django.db import transaction

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    res = list(LayerItem.objects.filter(name=options.layer)[:1])
    if res:
        layer = res[0]
    else:
        layer = LayerItem()
        layer.name = options.layer
        layer.status = 'P'
        layer.layer_type = 'M'
        layer.summary = 'OE-Classic'
        layer.description = 'OpenEmbedded-Classic'
        layer.vcs_url =  'git://git.openembedded.org/openembedded'
        layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded'
        layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
        layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
        layer.classic = True
        layer.save()

    layerbranch = layer.get_layerbranch(options.branch)
    if not layerbranch:
        # LayerBranch doesn't exist for this branch, create it
        layerbranch = LayerBranch()
        layerbranch.layer = layer
        layerbranch.branch = branch
        layerbranch.save()

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)
    fetchedrepos = []
    failedrepos = []

    bitbakepath = args[1]
    oeclassicpath = args[2]

    confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic'))
    os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger)
    except recipeparse.RecipeParseError as e:
        logger.error(str(e))
        sys.exit(1)

    # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
    tinfoil.config_data.setVar('SUMMARY', '')
    # Clear the default value of DESCRIPTION so that we can see where it's not set
    tinfoil.config_data.setVar('DESCRIPTION', '')
    # Clear the default value of HOMEPAGE ('unknown')
    tinfoil.config_data.setVar('HOMEPAGE', '')

    transaction.enter_transaction_management()
    transaction.managed(True)
    try:
        layerdir_start = os.path.normpath(oeclassicpath) + os.sep
        layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
        layermachines = Machine.objects.filter(layerbranch=layerbranch)
        layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
        layerclasses = BBClass.objects.filter(layerbranch=layerbranch)

        try:
            config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
        except recipeparse.RecipeParseError as e:
            logger.error(str(e))
            transaction.rollback()
            sys.exit(1)

        layerrecipes.delete()
        layermachines.delete()
        layerappends.delete()
        layerclasses.delete()
        for root, dirs, files in os.walk(oeclassicpath):
            if '.git' in dirs:
                dirs.remove('.git')
            for f in files:
                fullpath = os.path.join(root, f)
                (typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
                if typename == 'recipe':
                    recipe = ClassicRecipe()
                    recipe.layerbranch = layerbranch
                    recipe.filename = filename
                    recipe.filepath = filepath
                    update_recipe_file(config_data_copy, root, recipe, layerdir_start, oeclassicpath)
                    recipe.save()

        layerbranch.vcs_last_fetch = datetime.now()
        layerbranch.save()

        if options.dryrun:
            transaction.rollback()
        else:
            transaction.commit()
    except:
        import traceback
        traceback.print_exc()
        transaction.rollback()
    finally:
        transaction.leave_transaction_management()

    shutil.rmtree(tempdir)
    sys.exit(0)
예제 #3
0
def main():

    parser = optparse.OptionParser(usage="""
    %prog [options] <bitbakepath> <oeclassicpath>""")

    parser.add_option("-b",
                      "--branch",
                      help="Specify branch to import into",
                      action="store",
                      dest="branch",
                      default='oe-classic')
    parser.add_option("-l",
                      "--layer",
                      help="Specify layer to import into",
                      action="store",
                      dest="layer",
                      default='oe-classic')
    parser.add_option("-n",
                      "--dry-run",
                      help="Don't write any data back to the database",
                      action="store_true",
                      dest="dryrun")
    parser.add_option("-d",
                      "--debug",
                      help="Enable debug output",
                      action="store_const",
                      const=logging.DEBUG,
                      dest="loglevel",
                      default=logging.INFO)
    parser.add_option("-q",
                      "--quiet",
                      help="Hide all output except error messages",
                      action="store_const",
                      const=logging.ERROR,
                      dest="loglevel")

    options, args = parser.parse_args(sys.argv)
    if len(args) < 3:
        logger.error('You must specify bitbakepath and oeclassicpath')
        parser.print_help()
        sys.exit(1)
    if len(args) > 3:
        logger.error('unexpected argument "%s"' % args[3])
        parser.print_help()
        sys.exit(1)

    utils.setup_django()
    import settings
    from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
    from django.db import transaction

    logger.setLevel(options.loglevel)

    branch = utils.get_branch(options.branch)
    if not branch:
        logger.error("Specified branch %s is not valid" % options.branch)
        sys.exit(1)

    res = list(LayerItem.objects.filter(name=options.layer)[:1])
    if res:
        layer = res[0]
    else:
        layer = LayerItem()
        layer.name = options.layer
        layer.status = 'P'
        layer.layer_type = 'M'
        layer.summary = 'OE-Classic'
        layer.description = 'OpenEmbedded-Classic'
        layer.vcs_url = 'git://git.openembedded.org/openembedded'
        layer.vcs_web_url = 'http://cgit.openembedded.org/openembedded'
        layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
        layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
        layer.comparison = True
        layer.save()

    layerbranch = layer.get_layerbranch(options.branch)
    if not layerbranch:
        # LayerBranch doesn't exist for this branch, create it
        layerbranch = LayerBranch()
        layerbranch.layer = layer
        layerbranch.branch = branch
        layerbranch.save()

    fetchdir = settings.LAYER_FETCH_DIR
    if not fetchdir:
        logger.error("Please set LAYER_FETCH_DIR in settings.py")
        sys.exit(1)

    if not os.path.exists(fetchdir):
        os.makedirs(fetchdir)
    fetchedrepos = []
    failedrepos = []

    bitbakepath = args[1]
    oeclassicpath = args[2]

    confparentdir = os.path.abspath(
        os.path.join(os.path.dirname(os.path.abspath(__file__)),
                     '../../oe-classic'))
    os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
    try:
        (tinfoil, tempdir) = recipeparse.init_parser(settings,
                                                     branch,
                                                     bitbakepath,
                                                     nocheckout=True,
                                                     classic=True,
                                                     logger=logger)
    except recipeparse.RecipeParseError as e:
        logger.error(str(e))
        sys.exit(1)

    # Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
    tinfoil.config_data.setVar('SUMMARY', '')
    # Clear the default value of DESCRIPTION so that we can see where it's not set
    tinfoil.config_data.setVar('DESCRIPTION', '')
    # Clear the default value of HOMEPAGE ('unknown')
    tinfoil.config_data.setVar('HOMEPAGE', '')

    try:
        with transaction.atomic():
            layerdir_start = os.path.normpath(oeclassicpath) + os.sep
            layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
            layermachines = Machine.objects.filter(layerbranch=layerbranch)
            layerdistros = Distro.objects.filter(layerbranch=layerbranch)
            layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
            layerclasses = BBClass.objects.filter(layerbranch=layerbranch)

            try:
                config_data_copy = recipeparse.setup_layer(
                    tinfoil.config_data, fetchdir, oeclassicpath, layer,
                    layerbranch, logger)
            except recipeparse.RecipeParseError as e:
                logger.error(str(e))
                sys.exit(1)

            layerrecipes.delete()
            layermachines.delete()
            layerdistros.delete()
            layerappends.delete()
            layerclasses.delete()
            for root, dirs, files in os.walk(oeclassicpath):
                if '.git' in dirs:
                    dirs.remove('.git')
                for f in files:
                    fullpath = os.path.join(root, f)
                    (typename, filepath,
                     filename) = recipeparse.detect_file_type(
                         fullpath, layerdir_start)
                    if typename == 'recipe':
                        recipe = ClassicRecipe()
                        recipe.layerbranch = layerbranch
                        recipe.filename = filename
                        recipe.filepath = filepath
                        update_recipe_file(tinfoil, config_data_copy, root,
                                           recipe, layerdir_start,
                                           oeclassicpath)
                        recipe.save()

            layerbranch.vcs_last_fetch = datetime.now()
            layerbranch.save()

            if options.dryrun:
                raise DryRunRollbackException()
    except DryRunRollbackException:
        pass
    except:
        import traceback
        traceback.print_exc()
    finally:
        tinfoil.shutdown()

    shutil.rmtree(tempdir)
    sys.exit(0)
예제 #4
0
def main():

    parser = optparse.OptionParser(
        usage = """
    %prog [options]""")

    options, args = parser.parse_args(sys.argv)

    utils.setup_django()
    from layerindex.models import LayerItem, LayerBranch, LayerDependency
    from django.db import transaction

    import httplib
    conn = httplib.HTTPConnection("www.openembedded.org")
    conn.request("GET", "/wiki/LayerIndex?action=raw")
    resp = conn.getresponse()
    if resp.status in [200, 302]:
        data = resp.read()
        in_table = False
        layer_type = 'M'
        nowiki_re = re.compile(r'</?nowiki>')
        link_re = re.compile(r'\[(http.*) +link\]')
        readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
        master_branch = utils.get_branch('master')
        core_layer = None
        transaction.enter_transaction_management()
        transaction.managed(True)
        try:
            for line in data.splitlines():
                if line.startswith('{|'):
                    in_table = True
                    continue
                if in_table:
                    if line.startswith('|}'):
                        # We're done
                        break
                    elif line.startswith('!'):
                        section = line.split('|', 1)[1].strip("'")
                        if section.startswith('Base'):
                            layer_type = 'A'
                        elif section.startswith('Board'):
                            layer_type = 'B'
                        elif section.startswith('Software'):
                            layer_type = 'S'
                        elif section.startswith('Distribution'):
                            layer_type = 'D'
                        else:
                            layer_type = 'M'
                    elif not line.startswith('|-'):
                        if line.startswith("|| ''"):
                            continue
                        fields = line.split('||')
                        layer = LayerItem()
                        layer.name = fields[1].strip()
                        if ' ' in layer.name:
                            logger.warn('Skipping layer %s - name invalid' % layer.name)
                            continue
                        logger.info('Adding layer %s' % layer.name)
                        layer.status = 'P'
                        layer.layer_type = layer_type
                        layer.summary = fields[2].strip()
                        layer.description = layer.summary
                        if len(fields) > 6:
                            res = link_re.match(fields[6].strip())
                            if res:
                                link = res.groups(1)[0].strip()
                                if link.endswith('/README') or readme_re.search(link):
                                    link = 'README'
                                layer.usage_url = link

                        repoval = nowiki_re.sub('', fields[4]).strip()
                        layer.vcs_url = repoval
                        if repoval.startswith('git://git.openembedded.org/'):
                            reponame = re.sub('^.*/', '', repoval)
                            layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
                            layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
                            layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
                        elif 'git.yoctoproject.org/' in repoval:
                            reponame = re.sub('^.*/', '', repoval)
                            layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
                            layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
                            layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
                        elif 'github.com/' in repoval:
                            reponame = re.sub('^.*github.com/', '', repoval)
                            reponame = re.sub('.git$', '', reponame)
                            layer.vcs_web_url = 'http://github.com/' + reponame
                            layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
                            layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
                        elif 'gitorious.org/' in repoval:
                            reponame = re.sub('^.*gitorious.org/', '', repoval)
                            reponame = re.sub('.git$', '', reponame)
                            layer.vcs_web_url = 'http://gitorious.org/' + reponame
                            layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
                            layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
                        elif 'bitbucket.org/' in repoval:
                            reponame = re.sub('^.*bitbucket.org/', '', repoval)
                            reponame = re.sub('.git$', '', reponame)
                            layer.vcs_web_url = 'http://bitbucket.org/' + reponame
                            layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
                            layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
                        elif '.git' in repoval:
                            res = link_re.match(fields[5].strip())
                            layer.vcs_web_url = res.groups(1)[0]
                            layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
                            layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)

                        layer.save()
                        layerbranch = LayerBranch()
                        layerbranch.layer = layer
                        layerbranch.branch = master_branch
                        layerbranch.vcs_subdir = fields[3].strip()
                        layerbranch.save()
                        if layer.name != 'openembedded-core':
                            if not core_layer:
                                core_layer = utils.get_layer('openembedded-core')
                            if core_layer:
                                layerdep = LayerDependency()
                                layerdep.layerbranch = layerbranch
                                layerdep.dependency = core_layer
                                layerdep.save()
            transaction.commit()
        except:
            transaction.rollback()
            raise
        finally:
            transaction.leave_transaction_management()
    else:
        logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))

    sys.exit(0)