def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, LayerDependency
from django.db import transaction
import httplib
conn = httplib.HTTPConnection("www.openembedded.org")
conn.request("GET", "/wiki/LayerIndex?action=raw")
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
layer_type = 'M'
nowiki_re = re.compile(r'</?nowiki>')
link_re = re.compile(r'\[(http.*) +link\]')
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master')
core_layer = None
with transaction.atomic():
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
if in_table:
if line.startswith('|}'):
# We're done
break
elif line.startswith('!'):
section = line.split('|', 1)[1].strip("'")
if section.startswith('Base'):
layer_type = 'A'
elif section.startswith('Board'):
layer_type = 'B'
elif section.startswith('Software'):
layer_type = 'S'
elif section.startswith('Distribution'):
layer_type = 'D'
else:
layer_type = 'M'
elif not line.startswith('|-'):
if line.startswith("|| ''"):
continue
fields = line.split('||')
layer = LayerItem()
layer.name = fields[1].strip()
if ' ' in layer.name:
logger.warn('Skipping layer %s - name invalid' % layer.name)
continue
logger.info('Adding layer %s' % layer.name)
layer.status = 'P'
layer.layer_type = layer_type
layer.summary = fields[2].strip()
layer.description = layer.summary
if len(fields) > 6:
res = link_re.match(fields[6].strip())
if res:
link = res.groups(1)[0].strip()
if link.endswith('/README') or readme_re.search(link):
link = 'README'
layer.usage_url = link
repoval = nowiki_re.sub('', fields[4]).strip()
layer.vcs_url = repoval
if repoval.startswith('git://git.openembedded.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://cgit.openembedded.org/' + reponame
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_commit_url = 'http://cgit.openembedded.org/' + reponame + '/commit/?id=%hash%'
elif repoval.startswith('git://git.yoctoproject.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_commit_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/commit/?id=%hash%'
elif repoval.startswith('git://github.com/') or repoval.startswith('http://github.com/') or repoval.startswith('https://github.com/'):
reponame = re.sub('^.*github.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://github.com/' + reponame
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
layer.vcs_web_commit_url = 'http://github.com/' + reponame + '/commit/%hash%'
elif repoval.startswith('git://gitlab.com/') or repoval.startswith('http://gitlab.com/') or repoval.startswith('https://gitlab.com/'):
reponame = re.sub('^.*gitlab.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://gitlab.com/' + reponame
layer.vcs_web_tree_base_url = 'http://gitlab.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://gitlab.com/' + reponame + '/blob/%branch%/'
layer.vcs_web_commit_url = 'http://gitlab.com/' + reponame + '/commit/%hash%'
elif repoval.startswith('git://bitbucket.org/') or repoval.startswith('http://bitbucket.org/') or repoval.startswith('https://bitbucket.org/'):
reponame = re.sub('^.*bitbucket.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_commit_url = 'http://bitbucket.org/' + reponame + '/commits/%hash%'
elif '.git' in repoval:
res = link_re.match(fields[5].strip())
layer.vcs_web_url = res.groups(1)[0]
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=commit;h=%hash%', layer.vcs_web_url)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
layerbranch.vcs_subdir = fields[3].strip()
layerbranch.save()
if layer.name != 'openembedded-core':
if not core_layer:
core_layer = utils.get_layer('openembedded-core')
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options] <bitbakepath> <oeclassicpath>""")
parser.add_option("-b", "--branch",
help = "Specify branch to import into",
action="store", dest="branch", default='oe-classic')
parser.add_option("-l", "--layer",
help = "Specify layer to import into",
action="store", dest="layer", default='oe-classic')
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 3:
logger.error('You must specify bitbakepath and oeclassicpath')
parser.print_help()
sys.exit(1)
if len(args) > 3:
logger.error('unexpected argument "%s"' % args[3])
parser.print_help()
sys.exit(1)
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
res = list(LayerItem.objects.filter(name=options.layer)[:1])
if res:
layer = res[0]
else:
layer = LayerItem()
layer.name = options.layer
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'OE-Classic'
layer.description = 'OpenEmbedded-Classic'
layer.vcs_url = 'git://git.openembedded.org/openembedded'
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded'
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
layer.classic = True
layer.save()
layerbranch = layer.get_layerbranch(options.branch)
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
bitbakepath = args[1]
oeclassicpath = args[2]
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
transaction.enter_transaction_management()
transaction.managed(True)
try:
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try:
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
transaction.rollback()
sys.exit(1)
layerrecipes.delete()
layermachines.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs:
dirs.remove('.git')
for f in files:
fullpath = os.path.join(root, f)
(typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
if typename == 'recipe':
recipe = ClassicRecipe()
recipe.layerbranch = layerbranch
recipe.filename = filename
recipe.filepath = filepath
update_recipe_file(config_data_copy, root, recipe, layerdir_start, oeclassicpath)
recipe.save()
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
except:
import traceback
traceback.print_exc()
transaction.rollback()
finally:
transaction.leave_transaction_management()
shutil.rmtree(tempdir)
sys.exit(0)
def main():
parser = optparse.OptionParser(usage="""
%prog [options] <bitbakepath> <oeclassicpath>""")
parser.add_option("-b",
"--branch",
help="Specify branch to import into",
action="store",
dest="branch",
default='oe-classic')
parser.add_option("-l",
"--layer",
help="Specify layer to import into",
action="store",
dest="layer",
default='oe-classic')
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 3:
logger.error('You must specify bitbakepath and oeclassicpath')
parser.print_help()
sys.exit(1)
if len(args) > 3:
logger.error('unexpected argument "%s"' % args[3])
parser.print_help()
sys.exit(1)
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
res = list(LayerItem.objects.filter(name=options.layer)[:1])
if res:
layer = res[0]
else:
layer = LayerItem()
layer.name = options.layer
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'OE-Classic'
layer.description = 'OpenEmbedded-Classic'
layer.vcs_url = 'git://git.openembedded.org/openembedded'
layer.vcs_web_url = 'http://cgit.openembedded.org/openembedded'
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
layer.comparison = True
layer.save()
layerbranch = layer.get_layerbranch(options.branch)
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
bitbakepath = args[1]
oeclassicpath = args[2]
confparentdir = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../../oe-classic'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings,
branch,
bitbakepath,
nocheckout=True,
classic=True,
logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
try:
with transaction.atomic():
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try:
config_data_copy = recipeparse.setup_layer(
tinfoil.config_data, fetchdir, oeclassicpath, layer,
layerbranch, logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
layerrecipes.delete()
layermachines.delete()
layerdistros.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs:
dirs.remove('.git')
for f in files:
fullpath = os.path.join(root, f)
(typename, filepath,
filename) = recipeparse.detect_file_type(
fullpath, layerdir_start)
if typename == 'recipe':
recipe = ClassicRecipe()
recipe.layerbranch = layerbranch
recipe.filename = filename
recipe.filepath = filepath
update_recipe_file(tinfoil, config_data_copy, root,
recipe, layerdir_start,
oeclassicpath)
recipe.save()
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
except:
import traceback
traceback.print_exc()
finally:
tinfoil.shutdown()
shutil.rmtree(tempdir)
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options]""")
options, args = parser.parse_args(sys.argv)
utils.setup_django()
from layerindex.models import LayerItem, LayerBranch, LayerDependency
from django.db import transaction
import httplib
conn = httplib.HTTPConnection("www.openembedded.org")
conn.request("GET", "/wiki/LayerIndex?action=raw")
resp = conn.getresponse()
if resp.status in [200, 302]:
data = resp.read()
in_table = False
layer_type = 'M'
nowiki_re = re.compile(r'</?nowiki>')
link_re = re.compile(r'\[(http.*) +link\]')
readme_re = re.compile(r';f=[a-zA-Z0-9/-]*README;')
master_branch = utils.get_branch('master')
core_layer = None
transaction.enter_transaction_management()
transaction.managed(True)
try:
for line in data.splitlines():
if line.startswith('{|'):
in_table = True
continue
if in_table:
if line.startswith('|}'):
# We're done
break
elif line.startswith('!'):
section = line.split('|', 1)[1].strip("'")
if section.startswith('Base'):
layer_type = 'A'
elif section.startswith('Board'):
layer_type = 'B'
elif section.startswith('Software'):
layer_type = 'S'
elif section.startswith('Distribution'):
layer_type = 'D'
else:
layer_type = 'M'
elif not line.startswith('|-'):
if line.startswith("|| ''"):
continue
fields = line.split('||')
layer = LayerItem()
layer.name = fields[1].strip()
if ' ' in layer.name:
logger.warn('Skipping layer %s - name invalid' % layer.name)
continue
logger.info('Adding layer %s' % layer.name)
layer.status = 'P'
layer.layer_type = layer_type
layer.summary = fields[2].strip()
layer.description = layer.summary
if len(fields) > 6:
res = link_re.match(fields[6].strip())
if res:
link = res.groups(1)[0].strip()
if link.endswith('/README') or readme_re.search(link):
link = 'README'
layer.usage_url = link
repoval = nowiki_re.sub('', fields[4]).strip()
layer.vcs_url = repoval
if repoval.startswith('git://git.openembedded.org/'):
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
elif 'git.yoctoproject.org/' in repoval:
reponame = re.sub('^.*/', '', repoval)
layer.vcs_web_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame
layer.vcs_web_tree_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
layer.vcs_web_file_base_url = 'http://git.yoctoproject.org/cgit/cgit.cgi/' + reponame + '/tree/%path%?h=%branch%'
elif 'github.com/' in repoval:
reponame = re.sub('^.*github.com/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://github.com/' + reponame
layer.vcs_web_tree_base_url = 'http://github.com/' + reponame + '/tree/%branch%/'
layer.vcs_web_file_base_url = 'http://github.com/' + reponame + '/blob/%branch%/'
elif 'gitorious.org/' in repoval:
reponame = re.sub('^.*gitorious.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://gitorious.org/' + reponame
layer.vcs_web_tree_base_url = 'http://gitorious.org/' + reponame + '/trees/%branch%/'
layer.vcs_web_file_base_url = 'http://gitorious.org/' + reponame + '/blobs/%branch%/'
elif 'bitbucket.org/' in repoval:
reponame = re.sub('^.*bitbucket.org/', '', repoval)
reponame = re.sub('.git$', '', reponame)
layer.vcs_web_url = 'http://bitbucket.org/' + reponame
layer.vcs_web_tree_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
layer.vcs_web_file_base_url = 'http://bitbucket.org/' + reponame + '/src/%branch%/%path%?at=%branch%'
elif '.git' in repoval:
res = link_re.match(fields[5].strip())
layer.vcs_web_url = res.groups(1)[0]
layer.vcs_web_tree_base_url = re.sub(r'\.git.*', '.git;a=tree;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.vcs_web_file_base_url = re.sub(r'\.git.*', '.git;a=blob;f=%path%;hb=%branch%', layer.vcs_web_url)
layer.save()
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = master_branch
layerbranch.vcs_subdir = fields[3].strip()
layerbranch.save()
if layer.name != 'openembedded-core':
if not core_layer:
core_layer = utils.get_layer('openembedded-core')
if core_layer:
layerdep = LayerDependency()
layerdep.layerbranch = layerbranch
layerdep.dependency = core_layer
layerdep.save()
transaction.commit()
except:
transaction.rollback()
raise
finally:
transaction.leave_transaction_management()
else:
logger.error('Fetch failed: %d: %s' % (resp.status, resp.reason))
sys.exit(0)
