def main():
if LooseVersion(git.__version__) < '0.3.1':
logger.error(
"Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script"
)
sys.exit(1)
parser = optparse.OptionParser(usage="""
%prog [options]""")
parser.add_option("-b",
"--branch",
help="Specify branch to update",
action="store",
dest="branch",
default='master')
parser.add_option("-l",
"--layer",
help="Layer to update",
action="store",
dest="layer")
parser.add_option(
"-r",
"--reload",
help="Reload recipe data instead of updating since last update",
action="store_true",
dest="reload")
parser.add_option(
"",
"--fullreload",
help="Discard existing recipe data and fetch it from scratch",
action="store_true",
dest="fullreload")
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("",
"--nocheckout",
help="Don't check out branches",
action="store_true",
dest="nocheckout")
parser.add_option("",
"--stop-on-error",
help="Stop on first parsing error",
action="store_true",
default=False,
dest="stop_on_error")
parser.add_option("-i",
"--initial",
help="Print initial values parsed from layer.conf only",
action="store_true")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
parser.add_option(
"",
"--keep-temp",
help="Preserve temporary directory at the end instead of deleting it",
action="store_true")
options, args = parser.parse_args(sys.argv)
if len(args) > 1:
logger.error('unexpected argument "%s"' % args[1])
parser.print_help()
sys.exit(1)
if options.fullreload:
options.reload = True
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, Distro, BBAppend, BBClass, IncFile
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
bitbakepath = os.path.join(fetchdir, 'bitbake')
layer = utils.get_layer(options.layer)
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
layerbranch = layer.get_layerbranch(options.branch)
branchname = options.branch
branchdesc = options.branch
if layerbranch:
if layerbranch.actual_branch:
branchname = layerbranch.actual_branch
branchdesc = "%s (%s)" % (options.branch, branchname)
# Collect repo info
repo = git.Repo(repodir)
if repo.bare:
logger.error('Repository %s is bare, not supported' % repodir)
sys.exit(1)
topcommit = repo.commit('origin/%s' % branchname)
if options.nocheckout:
topcommit = repo.commit('HEAD')
tinfoil = None
tempdir = None
try:
with transaction.atomic():
newbranch = False
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
newbranch = True
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch_source = layer.get_layerbranch(branch)
if not layerbranch_source:
layerbranch_source = layer.get_layerbranch(None)
if layerbranch_source:
layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
layerbranch.save()
if layerbranch_source:
for maintainer in layerbranch_source.layermaintainer_set.all(
):
maintainer.pk = None
maintainer.id = None
maintainer.layerbranch = layerbranch
maintainer.save()
if layerbranch.vcs_subdir and not options.nocheckout:
# Find latest commit in subdirectory
# A bit odd to do it this way but apparently there's no other way in the GitPython API
topcommit = next(
repo.iter_commits('origin/%s' % branchname,
paths=layerbranch.vcs_subdir), None)
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
layerdir_start = os.path.normpath(layerdir) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
layerincfiles = IncFile.objects.filter(layerbranch=layerbranch)
if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload or options.initial:
# Check out appropriate branch
if not options.nocheckout:
utils.checkout_layer_branch(layerbranch,
repodir,
logger=logger)
logger.info("Collecting data for layer %s on branch %s" %
(layer.name, branchdesc))
try:
(tinfoil, tempdir) = recipeparse.init_parser(
settings,
branch,
bitbakepath,
nocheckout=options.nocheckout,
logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
logger.debug('Using temp directory %s' % tempdir)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
# Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
# why won't they just fix that?!)
tinfoil.config_data.setVar('LICENSE', '')
layerconfparser = layerconfparse.LayerConfParse(
logger=logger, tinfoil=tinfoil)
layer_config_data = layerconfparser.parse_layer(layerdir)
if not layer_config_data:
logger.info(
"Skipping update of layer %s for branch %s - conf/layer.conf may have parse issues"
% (layer.name, branchdesc))
layerconfparser.shutdown()
sys.exit(1)
utils.set_layerbranch_collection_version(layerbranch,
layer_config_data,
logger=logger)
if options.initial:
# Use print() rather than logger.info() since "-q" makes it print nothing.
for i in [
"BBFILE_COLLECTIONS", "LAYERVERSION",
"LAYERDEPENDS", "LAYERRECOMMENDS"
]:
print(
'%s = "%s"' % (i,
utils.get_layer_var(
layer_config_data, i, logger)))
sys.exit(0)
# Set up for recording patch info
utils.setup_core_layer_sys_path(settings, branch.name)
skip_patches = False
try:
import oe.recipeutils
except ImportError:
logger.warn(
'Failed to find lib/oe/recipeutils.py in layers - patch information will not be collected'
)
skip_patches = True
utils.add_dependencies(layerbranch,
layer_config_data,
logger=logger)
utils.add_recommends(layerbranch,
layer_config_data,
logger=logger)
layerbranch.save()
try:
config_data_copy = recipeparse.setup_layer(
tinfoil.config_data, fetchdir, layerdir, layer,
layerbranch, logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
if layerbranch.vcs_last_rev and not options.reload:
try:
diff = repo.commit(
layerbranch.vcs_last_rev).diff(topcommit)
except Exception as e:
logger.warn(
"Unable to get diff from last commit hash for layer %s - falling back to slow update: %s"
% (layer.name, str(e)))
diff = None
else:
diff = None
# We handle recipes specially to try to preserve the same id
# when recipe upgrades happen (so that if a user bookmarks a
# recipe page it remains valid)
layerrecipes_delete = []
layerrecipes_add = []
# Check if any paths should be ignored because there are layers within this layer
removedirs = []
for root, dirs, files in os.walk(layerdir):
for diritem in dirs:
if os.path.exists(
os.path.join(root, diritem, 'conf',
'layer.conf')):
removedirs.append(
os.path.join(root, diritem) + os.sep)
if diff:
# Apply git changes to existing recipe list
if layerbranch.vcs_subdir:
subdir_start = os.path.normpath(
layerbranch.vcs_subdir) + os.sep
else:
subdir_start = ""
updatedrecipes = set()
dirtyrecipes = set()
other_deletes = []
other_adds = []
for diffitem in diff.iter_change_type('R'):
oldpath = diffitem.a_blob.path
newpath = diffitem.b_blob.path
skip = False
for removedir in removedirs:
# FIXME what about files moved into removedirs?
if oldpath.startswith(removedir):
skip = True
break
if skip:
continue
if oldpath.startswith(subdir_start):
if not newpath.startswith(subdir_start):
logger.debug(
"Treating rename of %s to %s as a delete since new path is outside layer"
% (oldpath, newpath))
other_deletes.append(diffitem)
continue
(oldtypename, oldfilepath,
oldfilename) = recipeparse.detect_file_type(
oldpath, subdir_start)
(newtypename, newfilepath,
newfilename) = recipeparse.detect_file_type(
newpath, subdir_start)
if oldtypename != newtypename:
# This is most likely to be a .inc file renamed to a .bb - and since
# there may be another recipe deleted at the same time we probably want
# to consider that, so just treat it as a delete and an add
logger.debug(
"Treating rename of %s to %s as a delete and add (since type changed)"
% (oldpath, newpath))
other_deletes.append(diffitem)
other_adds.append(diffitem)
elif oldtypename == 'recipe':
results = layerrecipes.filter(
filepath=oldfilepath).filter(
filename=oldfilename)
if len(results):
recipe = results[0]
logger.debug("Rename recipe %s to %s" %
(recipe, newpath))
recipe.filepath = newfilepath
recipe.filename = newfilename
recipe.save()
update_recipe_file(
tinfoil, config_data_copy,
os.path.join(layerdir, newfilepath),
recipe, layerdir_start, repodir,
options.stop_on_error, skip_patches)
updatedrecipes.add(
os.path.join(oldfilepath, oldfilename))
updatedrecipes.add(
os.path.join(newfilepath, newfilename))
else:
logger.warn(
"Renamed recipe %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'bbappend':
results = layerappends.filter(
filepath=oldfilepath).filter(
filename=oldfilename)
if len(results):
logger.debug(
"Rename bbappend %s to %s" %
(results[0],
os.path.join(newfilepath,
newfilename)))
results[0].filepath = newfilepath
results[0].filename = newfilename
results[0].save()
else:
logger.warn(
"Renamed bbappend %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'machine':
results = layermachines.filter(
name=oldfilename)
if len(results):
logger.debug("Rename machine %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed machine %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'distro':
results = layerdistros.filter(name=oldfilename)
if len(results):
logger.debug("Rename distro %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed distro %s could not be found"
% oldpath)
other_adds.append(diffitem)
elif oldtypename == 'bbclass':
results = layerclasses.filter(name=oldfilename)
if len(results):
logger.debug("Rename class %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed class %s could not be found" %
oldpath)
other_adds.append(diffitem)
elif oldtypename == 'incfile':
results = layerincfiles.filter(
path=os.path.join(oldfilepath,
oldfilename))
if len(results):
logger.debug("Rename inc file %s to %s" %
(results[0], newfilename))
results[0].name = newfilename
results[0].save()
else:
logger.warn(
"Renamed inc file %s could not be found"
% oldpath)
other_adds.append(diffitem)
deps = RecipeFileDependency.objects.filter(
layerbranch=layerbranch).filter(path=oldpath)
for dep in deps:
dirtyrecipes.add(dep.recipe)
for diffitem in itertools.chain(diff.iter_change_type('D'),
other_deletes):
path = diffitem.a_blob.path
if path.startswith(subdir_start):
skip = False
for removedir in removedirs:
if path.startswith(removedir):
skip = True
break
if skip:
continue
(typename, filepath,
filename) = recipeparse.detect_file_type(
path, subdir_start)
if typename == 'recipe':
values = layerrecipes.filter(
filepath=filepath).filter(
filename=filename).values(
'id', 'filepath', 'filename', 'pn')
if len(values):
layerrecipes_delete.append(values[0])
logger.debug("Mark %s for deletion" %
values[0])
updatedrecipes.add(
os.path.join(values[0]['filepath'],
values[0]['filename']))
else:
logger.warn(
"Deleted recipe %s could not be found"
% path)
elif typename == 'bbappend':
layerappends.filter(filepath=filepath).filter(
filename=filename).delete()
elif typename == 'machine':
layermachines.filter(name=filename).delete()
elif typename == 'distro':
layerdistros.filter(name=filename).delete()
elif typename == 'bbclass':
layerclasses.filter(name=filename).delete()
elif typename == 'incfile':
layerincfiles.filter(path=os.path.join(
filepath, filename)).delete()
for diffitem in itertools.chain(diff.iter_change_type('A'),
other_adds):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
skip = False
for removedir in removedirs:
if path.startswith(removedir):
skip = True
break
if skip:
continue
(typename, filepath,
filename) = recipeparse.detect_file_type(
path, subdir_start)
if typename == 'recipe':
layerrecipes_add.append(
os.path.join(repodir, path))
logger.debug("Mark %s for addition" % path)
updatedrecipes.add(
os.path.join(filepath, filename))
elif typename == 'bbappend':
append = BBAppend()
append.layerbranch = layerbranch
append.filename = filename
append.filepath = filepath
append.save()
elif typename == 'machine':
machine = Machine()
machine.layerbranch = layerbranch
machine.name = filename
update_machine_conf_file(
os.path.join(repodir, path), machine)
machine.save()
elif typename == 'distro':
distro = Distro()
distro.layerbranch = layerbranch
distro.name = filename
update_distro_conf_file(
os.path.join(repodir, path), distro,
config_data_copy)
distro.save()
elif typename == 'bbclass':
bbclass = BBClass()
bbclass.layerbranch = layerbranch
bbclass.name = filename
bbclass.save()
elif typename == 'incfile':
incfile = IncFile()
incfile.layerbranch = layerbranch
incfile.path = os.path.join(filepath, filename)
incfile.save()
for diffitem in diff.iter_change_type('M'):
path = diffitem.b_blob.path
if path.startswith(subdir_start):
skip = False
for removedir in removedirs:
if path.startswith(removedir):
skip = True
break
if skip:
continue
(typename, filepath,
filename) = recipeparse.detect_file_type(
path, subdir_start)
if typename == 'recipe':
logger.debug("Mark %s for update" % path)
results = layerrecipes.filter(
filepath=filepath).filter(
filename=filename)[:1]
if results:
recipe = results[0]
update_recipe_file(
tinfoil, config_data_copy,
os.path.join(layerdir, filepath),
recipe, layerdir_start, repodir,
options.stop_on_error, skip_patches)
recipe.save()
updatedrecipes.add(recipe.full_path())
elif typename == 'machine':
results = layermachines.filter(name=filename)
if results:
machine = results[0]
update_machine_conf_file(
os.path.join(repodir, path), machine)
machine.save()
elif typename == 'distro':
results = layerdistros.filter(name=filename)
if results:
distro = results[0]
update_distro_conf_file(
os.path.join(repodir, path), distro,
config_data_copy)
distro.save()
deps = RecipeFileDependency.objects.filter(
layerbranch=layerbranch).filter(path=path)
for dep in deps:
dirtyrecipes.add(dep.recipe)
for recipe in dirtyrecipes:
if not recipe.full_path() in updatedrecipes:
update_recipe_file(
tinfoil, config_data_copy,
os.path.join(layerdir, recipe.filepath),
recipe, layerdir_start, repodir,
options.stop_on_error, skip_patches)
else:
# Collect recipe data from scratch
layerrecipe_fns = []
if options.fullreload:
layerrecipes.delete()
else:
# First, check which recipes still exist
layerrecipe_values = layerrecipes.values(
'id', 'filepath', 'filename', 'pn')
for v in layerrecipe_values:
if v['filepath'].startswith('../'):
# FIXME: These recipes were present due to a bug (not handling renames
# to paths outside the layer) - this can be removed at some point in the future
preserve = False
else:
root = os.path.join(layerdir, v['filepath'])
fullpath = os.path.join(root, v['filename'])
if os.path.exists(fullpath):
preserve = True
for removedir in removedirs:
if fullpath.startswith(removedir):
preserve = False
break
else:
preserve = False
if preserve:
# Recipe still exists, update it
results = layerrecipes.filter(id=v['id'])[:1]
recipe = results[0]
update_recipe_file(tinfoil, config_data_copy,
root, recipe,
layerdir_start, repodir,
options.stop_on_error,
skip_patches)
else:
# Recipe no longer exists, mark it for later on
layerrecipes_delete.append(v)
layerrecipe_fns.append(fullpath)
layermachines.delete()
layerdistros.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(layerdir):
if '.git' in dirs:
dirs.remove('.git')
for diritem in dirs[:]:
fullpath = os.path.join(root, diritem) + os.sep
if fullpath in removedirs:
dirs.remove(diritem)
for f in files:
fullpath = os.path.join(root, f)
(typename, _,
filename) = recipeparse.detect_file_type(
fullpath, layerdir_start)
if typename == 'recipe':
if fullpath not in layerrecipe_fns:
layerrecipes_add.append(fullpath)
elif typename == 'bbappend':
append = BBAppend()
append.layerbranch = layerbranch
append.filename = f
append.filepath = os.path.relpath(
root, layerdir)
append.save()
elif typename == 'machine':
machine = Machine()
machine.layerbranch = layerbranch
machine.name = filename
update_machine_conf_file(fullpath, machine)
machine.save()
elif typename == 'distro':
distro = Distro()
distro.layerbranch = layerbranch
distro.name = filename
update_distro_conf_file(
fullpath, distro, config_data_copy)
distro.save()
elif typename == 'bbclass':
bbclass = BBClass()
bbclass.layerbranch = layerbranch
bbclass.name = filename
bbclass.save()
elif typename == 'incfile':
incfile = IncFile()
incfile.layerbranch = layerbranch
incfile.path = os.path.relpath(
fullpath, layerdir)
incfile.save()
for added in layerrecipes_add:
# This is good enough without actually parsing the file
(pn, pv) = split_recipe_fn(added)
oldid = -1
for deleted in layerrecipes_delete:
if deleted['pn'] == pn:
oldid = deleted['id']
layerrecipes_delete.remove(deleted)
break
if oldid > -1:
# Reclaim a record we would have deleted
results = Recipe.objects.filter(id=oldid)[:1]
recipe = results[0]
logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
else:
# Create new record
logger.debug("Add new recipe %s" % added)
recipe = Recipe()
recipe.layerbranch = layerbranch
recipe.filename = os.path.basename(added)
root = os.path.dirname(added)
recipe.filepath = os.path.relpath(root, layerdir)
update_recipe_file(tinfoil, config_data_copy, root, recipe,
layerdir_start, repodir,
options.stop_on_error, skip_patches)
recipe.save()
for deleted in layerrecipes_delete:
logger.debug("Delete %s" % deleted)
results = Recipe.objects.filter(id=deleted['id'])[:1]
recipe = results[0]
recipe.delete()
# Save repo info
layerbranch.vcs_last_rev = topcommit.hexsha
layerbranch.vcs_last_commit = datetime.fromtimestamp(
topcommit.committed_date)
else:
logger.info("Layer %s is already up-to-date for branch %s" %
(layer.name, branchdesc))
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
raise DryRunRollbackException()
except KeyboardInterrupt:
logger.warn("Update interrupted, changes to %s rolled back" %
layer.name)
sys.exit(254)
except SystemExit:
raise
except DryRunRollbackException:
pass
except:
import traceback
logger.error(traceback.format_exc().rstrip())
sys.exit(1)
finally:
if tinfoil and (LooseVersion(bb.__version__) > LooseVersion("1.27")):
tinfoil.shutdown()
if tempdir:
if options.keep_temp:
logger.debug('Preserving temp directory %s' % tempdir)
else:
logger.debug('Deleting temp directory')
utils.rmtree_force(tempdir)
sys.exit(0)
def main():
parser = optparse.OptionParser(
usage = """
%prog [options] <bitbakepath> <oeclassicpath>""")
parser.add_option("-b", "--branch",
help = "Specify branch to import into",
action="store", dest="branch", default='oe-classic')
parser.add_option("-l", "--layer",
help = "Specify layer to import into",
action="store", dest="layer", default='oe-classic')
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 3:
logger.error('You must specify bitbakepath and oeclassicpath')
parser.print_help()
sys.exit(1)
if len(args) > 3:
logger.error('unexpected argument "%s"' % args[3])
parser.print_help()
sys.exit(1)
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
res = list(LayerItem.objects.filter(name=options.layer)[:1])
if res:
layer = res[0]
else:
layer = LayerItem()
layer.name = options.layer
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'OE-Classic'
layer.description = 'OpenEmbedded-Classic'
layer.vcs_url = 'git://git.openembedded.org/openembedded'
layer.vcs_web_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded'
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/cgit.cgi/openembedded/tree/%path%'
layer.classic = True
layer.save()
layerbranch = layer.get_layerbranch(options.branch)
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
bitbakepath = args[1]
oeclassicpath = args[2]
confparentdir = os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../oe-classic'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=True, classic=True, logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
transaction.enter_transaction_management()
transaction.managed(True)
try:
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try:
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, oeclassicpath, layer, layerbranch)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
transaction.rollback()
sys.exit(1)
layerrecipes.delete()
layermachines.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs:
dirs.remove('.git')
for f in files:
fullpath = os.path.join(root, f)
(typename, filepath, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
if typename == 'recipe':
recipe = ClassicRecipe()
recipe.layerbranch = layerbranch
recipe.filename = filename
recipe.filepath = filepath
update_recipe_file(config_data_copy, root, recipe, layerdir_start, oeclassicpath)
recipe.save()
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
except:
import traceback
traceback.print_exc()
transaction.rollback()
finally:
transaction.leave_transaction_management()
shutil.rmtree(tempdir)
sys.exit(0)
def main():
parser = optparse.OptionParser(usage="""
%prog [options] <bitbakepath> <oeclassicpath>""")
parser.add_option("-b",
"--branch",
help="Specify branch to import into",
action="store",
dest="branch",
default='oe-classic')
parser.add_option("-l",
"--layer",
help="Specify layer to import into",
action="store",
dest="layer",
default='oe-classic')
parser.add_option("-n",
"--dry-run",
help="Don't write any data back to the database",
action="store_true",
dest="dryrun")
parser.add_option("-d",
"--debug",
help="Enable debug output",
action="store_const",
const=logging.DEBUG,
dest="loglevel",
default=logging.INFO)
parser.add_option("-q",
"--quiet",
help="Hide all output except error messages",
action="store_const",
const=logging.ERROR,
dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) < 3:
logger.error('You must specify bitbakepath and oeclassicpath')
parser.print_help()
sys.exit(1)
if len(args) > 3:
logger.error('unexpected argument "%s"' % args[3])
parser.print_help()
sys.exit(1)
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, ClassicRecipe, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
res = list(LayerItem.objects.filter(name=options.layer)[:1])
if res:
layer = res[0]
else:
layer = LayerItem()
layer.name = options.layer
layer.status = 'P'
layer.layer_type = 'M'
layer.summary = 'OE-Classic'
layer.description = 'OpenEmbedded-Classic'
layer.vcs_url = 'git://git.openembedded.org/openembedded'
layer.vcs_web_url = 'http://cgit.openembedded.org/openembedded'
layer.vcs_web_tree_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
layer.vcs_web_file_base_url = 'http://cgit.openembedded.org/openembedded/tree/%path%'
layer.comparison = True
layer.save()
layerbranch = layer.get_layerbranch(options.branch)
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch.save()
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
bitbakepath = args[1]
oeclassicpath = args[2]
confparentdir = os.path.abspath(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'../../oe-classic'))
os.environ['BBPATH'] = str("%s:%s" % (confparentdir, oeclassicpath))
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings,
branch,
bitbakepath,
nocheckout=True,
classic=True,
logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
try:
with transaction.atomic():
layerdir_start = os.path.normpath(oeclassicpath) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerdistros = Distro.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
try:
config_data_copy = recipeparse.setup_layer(
tinfoil.config_data, fetchdir, oeclassicpath, layer,
layerbranch, logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
layerrecipes.delete()
layermachines.delete()
layerdistros.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(oeclassicpath):
if '.git' in dirs:
dirs.remove('.git')
for f in files:
fullpath = os.path.join(root, f)
(typename, filepath,
filename) = recipeparse.detect_file_type(
fullpath, layerdir_start)
if typename == 'recipe':
recipe = ClassicRecipe()
recipe.layerbranch = layerbranch
recipe.filename = filename
recipe.filepath = filepath
update_recipe_file(tinfoil, config_data_copy, root,
recipe, layerdir_start,
oeclassicpath)
recipe.save()
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
raise DryRunRollbackException()
except DryRunRollbackException:
pass
except:
import traceback
traceback.print_exc()
finally:
tinfoil.shutdown()
shutil.rmtree(tempdir)
sys.exit(0)
def main():
if LooseVersion(git.__version__) < '0.3.1':
logger.error("Version of GitPython is too old, please install GitPython (python-git) 0.3.1 or later in order to use this script")
sys.exit(1)
parser = optparse.OptionParser(
usage = """
%prog [options]""")
parser.add_option("-b", "--branch",
help = "Specify branch to update",
action="store", dest="branch", default='master')
parser.add_option("-l", "--layer",
help = "Specify layers to update (use commas to separate multiple). Default is all published layers.",
action="store", dest="layers")
parser.add_option("-r", "--reload",
help = "Reload recipe data instead of updating since last update",
action="store_true", dest="reload")
parser.add_option("", "--fullreload",
help = "Discard existing recipe data and fetch it from scratch",
action="store_true", dest="fullreload")
parser.add_option("-n", "--dry-run",
help = "Don't write any data back to the database",
action="store_true", dest="dryrun")
parser.add_option("-x", "--nofetch",
help = "Don't fetch repositories",
action="store_true", dest="nofetch")
parser.add_option("", "--nocheckout",
help = "Don't check out branches",
action="store_true", dest="nocheckout")
parser.add_option("-d", "--debug",
help = "Enable debug output",
action="store_const", const=logging.DEBUG, dest="loglevel", default=logging.INFO)
parser.add_option("-q", "--quiet",
help = "Hide all output except error messages",
action="store_const", const=logging.ERROR, dest="loglevel")
options, args = parser.parse_args(sys.argv)
if len(args) > 1:
logger.error('unexpected argument "%s"' % args[1])
parser.print_help()
sys.exit(1)
if options.fullreload:
options.reload = True
utils.setup_django()
import settings
from layerindex.models import LayerItem, LayerBranch, Recipe, RecipeFileDependency, Machine, BBAppend, BBClass
from django.db import transaction
logger.setLevel(options.loglevel)
branch = utils.get_branch(options.branch)
if not branch:
logger.error("Specified branch %s is not valid" % options.branch)
sys.exit(1)
fetchdir = settings.LAYER_FETCH_DIR
if not fetchdir:
logger.error("Please set LAYER_FETCH_DIR in settings.py")
sys.exit(1)
if options.layers:
layerquery = LayerItem.objects.filter(classic=False).filter(name__in=options.layers.split(','))
if layerquery.count() == 0:
logger.error('No layers matching specified query "%s"' % options.layers)
sys.exit(1)
else:
layerquery = LayerItem.objects.filter(classic=False).filter(status='P')
if layerquery.count() == 0:
logger.info("No published layers to update")
sys.exit(1)
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
fetchedrepos = []
failedrepos = []
lockfn = os.path.join(fetchdir, "layerindex.lock")
lockfile = utils.lock_file(lockfn)
if not lockfile:
logger.error("Layer index lock timeout expired")
sys.exit(1)
try:
bitbakepath = os.path.join(fetchdir, 'bitbake')
if not options.nofetch:
# Fetch latest metadata from repositories
for layer in layerquery:
# Handle multiple layers in a single repo
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
if not (layer.vcs_url in fetchedrepos or layer.vcs_url in failedrepos):
logger.info("Fetching remote repository %s" % layer.vcs_url)
out = None
try:
if not os.path.exists(repodir):
out = utils.runcmd("git clone %s %s" % (layer.vcs_url, urldir), fetchdir, logger=logger)
else:
out = utils.runcmd("git fetch", repodir, logger=logger)
except Exception as e:
logger.error("Fetch of layer %s failed: %s" % (layer.name, str(e)))
failedrepos.append(layer.vcs_url)
continue
fetchedrepos.append(layer.vcs_url)
if not fetchedrepos:
logger.error("No repositories could be fetched, exiting")
sys.exit(1)
logger.info("Fetching bitbake from remote repository %s" % settings.BITBAKE_REPO_URL)
if not os.path.exists(bitbakepath):
out = utils.runcmd("git clone %s %s" % (settings.BITBAKE_REPO_URL, 'bitbake'), fetchdir, logger=logger)
else:
out = utils.runcmd("git fetch", bitbakepath, logger=logger)
try:
(tinfoil, tempdir) = recipeparse.init_parser(settings, branch, bitbakepath, nocheckout=options.nocheckout, logger=logger)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
sys.exit(1)
# Clear the default value of SUMMARY so that we can use DESCRIPTION instead if it hasn't been set
tinfoil.config_data.setVar('SUMMARY', '')
# Clear the default value of DESCRIPTION so that we can see where it's not set
tinfoil.config_data.setVar('DESCRIPTION', '')
# Clear the default value of HOMEPAGE ('unknown')
tinfoil.config_data.setVar('HOMEPAGE', '')
# Set a blank value for LICENSE so that it doesn't cause the parser to die (e.g. with meta-ti -
# why won't they just fix that?!)
tinfoil.config_data.setVar('LICENSE', '')
# Process and extract data from each layer
for layer in layerquery:
transaction.enter_transaction_management()
transaction.managed(True)
try:
urldir = layer.get_fetch_dir()
repodir = os.path.join(fetchdir, urldir)
if layer.vcs_url in failedrepos:
logger.info("Skipping update of layer %s as fetch of repository %s failed" % (layer.name, layer.vcs_url))
transaction.rollback()
continue
layerbranch = layer.get_layerbranch(options.branch)
branchname = options.branch
branchdesc = options.branch
if layerbranch:
if layerbranch.actual_branch:
branchname = layerbranch.actual_branch
branchdesc = "%s (%s)" % (options.branch, branchname)
# Collect repo info
repo = git.Repo(repodir)
assert repo.bare == False
try:
if options.nocheckout:
topcommit = repo.commit('HEAD')
else:
topcommit = repo.commit('origin/%s' % branchname)
except:
if layerbranch:
logger.error("Failed update of layer %s - branch %s no longer exists" % (layer.name, branchdesc))
else:
logger.info("Skipping update of layer %s - branch %s doesn't exist" % (layer.name, branchdesc))
transaction.rollback()
continue
newbranch = False
if not layerbranch:
# LayerBranch doesn't exist for this branch, create it
newbranch = True
layerbranch = LayerBranch()
layerbranch.layer = layer
layerbranch.branch = branch
layerbranch_source = layer.get_layerbranch('master')
if not layerbranch_source:
layerbranch_source = layer.get_layerbranch(None)
if layerbranch_source:
layerbranch.vcs_subdir = layerbranch_source.vcs_subdir
layerbranch.save()
if layerbranch_source:
for maintainer in layerbranch_source.layermaintainer_set.all():
maintainer.pk = None
maintainer.id = None
maintainer.layerbranch = layerbranch
maintainer.save()
for dep in layerbranch_source.dependencies_set.all():
dep.pk = None
dep.id = None
dep.layerbranch = layerbranch
dep.save()
if layerbranch.vcs_subdir and not options.nocheckout:
# Find latest commit in subdirectory
# A bit odd to do it this way but apparently there's no other way in the GitPython API
topcommit = next(repo.iter_commits('origin/%s' % branchname, paths=layerbranch.vcs_subdir), None)
if not topcommit:
# This will error out if the directory is completely invalid or had never existed at this point
# If it previously existed but has since been deleted, you will get the revision where it was
# deleted - so we need to handle that case separately later
if newbranch:
logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
elif layerbranch.vcs_subdir:
logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
else:
logger.error("Failed to get last revision for layer %s on branch %s" % (layer.name, branchdesc))
transaction.rollback()
continue
layerdir = os.path.join(repodir, layerbranch.vcs_subdir)
layerdir_start = os.path.normpath(layerdir) + os.sep
layerrecipes = Recipe.objects.filter(layerbranch=layerbranch)
layermachines = Machine.objects.filter(layerbranch=layerbranch)
layerappends = BBAppend.objects.filter(layerbranch=layerbranch)
layerclasses = BBClass.objects.filter(layerbranch=layerbranch)
if layerbranch.vcs_last_rev != topcommit.hexsha or options.reload:
# Check out appropriate branch
if not options.nocheckout:
out = utils.runcmd("git checkout origin/%s" % branchname, repodir, logger=logger)
out = utils.runcmd("git clean -f -x", repodir, logger=logger)
if layerbranch.vcs_subdir and not os.path.exists(layerdir):
if newbranch:
logger.info("Skipping update of layer %s for branch %s - subdirectory %s does not exist on this branch" % (layer.name, branchdesc, layerbranch.vcs_subdir))
else:
logger.error("Subdirectory for layer %s does not exist on branch %s - if this is legitimate, the layer branch record should be deleted" % (layer.name, branchdesc))
transaction.rollback()
continue
if not os.path.exists(os.path.join(layerdir, 'conf/layer.conf')):
logger.error("conf/layer.conf not found for layer %s - is subdirectory set correctly?" % layer.name)
transaction.rollback()
continue
logger.info("Collecting data for layer %s on branch %s" % (layer.name, branchdesc))
try:
config_data_copy = recipeparse.setup_layer(tinfoil.config_data, fetchdir, layerdir, layer, layerbranch)
except recipeparse.RecipeParseError as e:
logger.error(str(e))
transaction.rollback()
continue
if layerbranch.vcs_last_rev and not options.reload:
try:
diff = repo.commit(layerbranch.vcs_last_rev).diff(topcommit)
except Exception as e:
logger.warn("Unable to get diff from last commit hash for layer %s - falling back to slow update: %s" % (layer.name, str(e)))
diff = None
else:
diff = None
# We handle recipes specially to try to preserve the same id
# when recipe upgrades happen (so that if a user bookmarks a
# recipe page it remains valid)
layerrecipes_delete = []
layerrecipes_add = []
# Check if any paths should be ignored because there are layers within this layer
removedirs = []
for root, dirs, files in os.walk(layerdir):
for d in dirs:
if os.path.exists(os.path.join(root, d, 'conf', 'layer.conf')):
removedirs.append(os.path.join(root, d) + os.sep)
if diff:
# Apply git changes to existing recipe list
if layerbranch.vcs_subdir:
subdir_start = os.path.normpath(layerbranch.vcs_subdir) + os.sep
else:
subdir_start = ""
updatedrecipes = set()
for d in diff.iter_change_type('D'):
path = d.a_blob.path
if path.startswith(subdir_start):
skip = False
for d in removedirs:
if path.startswith(d):
skip = True
break
if skip:
continue
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
if typename == 'recipe':
values = layerrecipes.filter(filepath=filepath).filter(filename=filename).values('id', 'filepath', 'filename', 'pn')
if len(values):
layerrecipes_delete.append(values[0])
logger.debug("Mark %s for deletion" % values[0])
updatedrecipes.add(os.path.join(values[0]['filepath'], values[0]['filename']))
else:
logger.warn("Deleted recipe %s could not be found" % path)
elif typename == 'bbappend':
layerappends.filter(filepath=filepath).filter(filename=filename).delete()
elif typename == 'machine':
layermachines.filter(name=filename).delete()
elif typename == 'bbclass':
layerclasses.filter(name=filename).delete()
for d in diff.iter_change_type('A'):
path = d.b_blob.path
if path.startswith(subdir_start):
skip = False
for d in removedirs:
if path.startswith(d):
skip = True
break
if skip:
continue
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
if typename == 'recipe':
layerrecipes_add.append(os.path.join(repodir, path))
logger.debug("Mark %s for addition" % path)
updatedrecipes.add(os.path.join(filepath, filename))
elif typename == 'bbappend':
append = BBAppend()
append.layerbranch = layerbranch
append.filename = filename
append.filepath = filepath
append.save()
elif typename == 'machine':
machine = Machine()
machine.layerbranch = layerbranch
machine.name = filename
update_machine_conf_file(os.path.join(repodir, path), machine)
machine.save()
elif typename == 'bbclass':
bbclass = BBClass()
bbclass.layerbranch = layerbranch
bbclass.name = filename
bbclass.save()
dirtyrecipes = set()
for d in diff.iter_change_type('M'):
path = d.a_blob.path
if path.startswith(subdir_start):
skip = False
for d in removedirs:
if path.startswith(d):
skip = True
break
if skip:
continue
(typename, filepath, filename) = recipeparse.detect_file_type(path, subdir_start)
if typename == 'recipe':
logger.debug("Mark %s for update" % path)
results = layerrecipes.filter(filepath=filepath).filter(filename=filename)[:1]
if results:
recipe = results[0]
update_recipe_file(config_data_copy, os.path.join(layerdir, filepath), recipe, layerdir_start, repodir)
recipe.save()
updatedrecipes.add(recipe.full_path())
elif typename == 'machine':
results = layermachines.filter(name=filename)
if results:
machine = results[0]
update_machine_conf_file(os.path.join(repodir, path), machine)
machine.save()
deps = RecipeFileDependency.objects.filter(layerbranch=layerbranch).filter(path=path)
for dep in deps:
dirtyrecipes.add(dep.recipe)
for recipe in dirtyrecipes:
if not recipe.full_path() in updatedrecipes:
update_recipe_file(config_data_copy, os.path.join(layerdir, recipe.filepath), recipe, layerdir_start, repodir)
else:
# Collect recipe data from scratch
layerrecipe_fns = []
if options.fullreload:
layerrecipes.delete()
else:
# First, check which recipes still exist
layerrecipe_values = layerrecipes.values('id', 'filepath', 'filename', 'pn')
for v in layerrecipe_values:
root = os.path.join(layerdir, v['filepath'])
fullpath = os.path.join(root, v['filename'])
preserve = True
if os.path.exists(fullpath):
for d in removedirs:
if fullpath.startswith(d):
preserve = False
break
else:
preserve = False
if preserve:
# Recipe still exists, update it
results = layerrecipes.filter(id=v['id'])[:1]
recipe = results[0]
update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
else:
# Recipe no longer exists, mark it for later on
layerrecipes_delete.append(v)
layerrecipe_fns.append(fullpath)
layermachines.delete()
layerappends.delete()
layerclasses.delete()
for root, dirs, files in os.walk(layerdir):
if '.git' in dirs:
dirs.remove('.git')
for d in dirs[:]:
fullpath = os.path.join(root, d) + os.sep
if fullpath in removedirs:
dirs.remove(d)
for f in files:
fullpath = os.path.join(root, f)
(typename, _, filename) = recipeparse.detect_file_type(fullpath, layerdir_start)
if typename == 'recipe':
if fullpath not in layerrecipe_fns:
layerrecipes_add.append(fullpath)
elif typename == 'bbappend':
append = BBAppend()
append.layerbranch = layerbranch
append.filename = f
append.filepath = os.path.relpath(root, layerdir)
append.save()
elif typename == 'machine':
machine = Machine()
machine.layerbranch = layerbranch
machine.name = filename
update_machine_conf_file(fullpath, machine)
machine.save()
elif typename == 'bbclass':
bbclass = BBClass()
bbclass.layerbranch = layerbranch
bbclass.name = filename
bbclass.save()
for added in layerrecipes_add:
# This is good enough without actually parsing the file
(pn, pv) = split_recipe_fn(added)
oldid = -1
for deleted in layerrecipes_delete:
if deleted['pn'] == pn:
oldid = deleted['id']
layerrecipes_delete.remove(deleted)
break
if oldid > -1:
# Reclaim a record we would have deleted
results = Recipe.objects.filter(id=oldid)[:1]
recipe = results[0]
logger.debug("Reclaim %s for %s %s" % (recipe, pn, pv))
else:
# Create new record
logger.debug("Add new recipe %s" % added)
recipe = Recipe()
recipe.layerbranch = layerbranch
recipe.filename = os.path.basename(added)
root = os.path.dirname(added)
recipe.filepath = os.path.relpath(root, layerdir)
update_recipe_file(config_data_copy, root, recipe, layerdir_start, repodir)
recipe.save()
for deleted in layerrecipes_delete:
logger.debug("Delete %s" % deleted)
results = Recipe.objects.filter(id=deleted['id'])[:1]
recipe = results[0]
recipe.delete()
# Save repo info
layerbranch.vcs_last_rev = topcommit.hexsha
layerbranch.vcs_last_commit = datetime.fromtimestamp(topcommit.committed_date)
else:
logger.info("Layer %s is already up-to-date for branch %s" % (layer.name, branchdesc))
layerbranch.vcs_last_fetch = datetime.now()
layerbranch.save()
if options.dryrun:
transaction.rollback()
else:
transaction.commit()
# Slightly hacky way of avoiding memory leaks
bb.event.ui_queue = []
bb.parse.parse_py.BBHandler.cached_statements = {}
bb.codeparser.codeparsercache = bb.codeparser.CodeParserCache()
if hasattr(bb.codeparser, 'codecache'):
bb.codeparser.codecache = bb.codeparser.SetCache()
bb.fetch._checksum_cache = bb.checksum.FileChecksumCache()
bb.fetch.urldata_cache = {}
bb.fetch.saved_headrevs = {}
bb.parse.__pkgsplit_cache__={}
bb.parse.__mtime_cache = {}
bb.parse.init_parser(tinfoil.config_data)
except KeyboardInterrupt:
transaction.rollback()
logger.warn("Update interrupted, changes to %s rolled back" % layer.name)
break
except:
import traceback
traceback.print_exc()
transaction.rollback()
finally:
transaction.leave_transaction_management()
finally:
utils.unlock_file(lockfile)
shutil.rmtree(tempdir)
sys.exit(0)
