def test_layerindex_store_split(self):
self.layerindex.store_layerindex('file://%s' % self.tempdir,
self.layerindex.indexes[0])
reload = layerindexlib.LayerIndex(self.d)
reload.load_layerindex('file://%s' % self.tempdir)
self.assertFalse(reload.is_empty(), msg="Layer index is empty")
for type in self.layerindex.indexes[0]._index:
if type == 'apilinks' or \
type == 'layerItems' or \
type in self.layerindex.indexes[0].config['local']:
continue
for id in getattr(self.layerindex.indexes[0], type):
self.logger.debug("type %s" % (type))
self.assertTrue(id in getattr(reload.indexes[0], type),
msg="Id number missing from reloaded data")
self.logger.debug(
"%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id],
getattr(reload.indexes[0], type)[id]))
self.assertEqual(getattr(self.layerindex.indexes[0], type)[id],
getattr(reload.indexes[0], type)[id],
msg="reloaded data does not match original")
def test_layerindex_store_file(self):
self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0])
self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex")
reload = layerindexlib.LayerIndex(self.d)
reload.load_layerindex('file://%s/file.json' % self.tempdir)
self.assertFalse(reload.is_empty(), msg="Layerindex is empty")
# Calculate layerItems in original index that should NOT be in reload
layerItemNames = []
for itemId in self.layerindex.indexes[0].layerItems:
layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name)
for layerBranchId in self.layerindex.indexes[0].layerBranches:
layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name)
for itemId in reload.indexes[0].layerItems:
self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been")
# Compare the original to what we wrote...
for type in self.layerindex.indexes[0]._index:
if type == 'apilinks' or \
type == 'layerItems' or \
type in self.layerindex.indexes[0].config['local']:
continue
for id in getattr(self.layerindex.indexes[0], type):
self.logger.debug(1, "type %s" % (type))
self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index")
self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id]))
self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
def setUp(self):
self.assertFalse(
os.environ.get("BB_SKIP_NETTESTS") == "yes",
msg="BB_SKIP_NETTESTS set, but we tried to test anyway")
LayersTest.setUp(self)
self.layerindex = layerindexlib.LayerIndex(self.d)
self.layerindex.load_layerindex(
'http://layers.openembedded.org/layerindex/api/;branch=sumo',
load=['layerDependencies'])
def setUp(self):
LayersTest.setUp(self)
# Note this is NOT a comprehensive test of cooker, as we can't easily
# configure the test data. But we can emulate the basics of the layer.conf
# files, so that is what we will do.
new_topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testdata")
new_bbpath = os.path.join(new_topdir, "build")
self.d.setVar('TOPDIR', new_topdir)
self.d.setVar('BBPATH', new_bbpath)
self.d = bb.parse.handle("%s/conf/bblayers.conf" % new_bbpath, self.d, True)
for layer in self.d.getVar('BBLAYERS').split():
self.d = bb.parse.handle("%s/conf/layer.conf" % layer, self.d, True)
self.layerindex = layerindexlib.LayerIndex(self.d)
self.layerindex.load_layerindex('cooker://', load=['layerDependencies'])
def do_layerindex_fetch(self, args):
"""Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf.
"""
def _construct_url(baseurls, branches):
urls = []
for baseurl in baseurls:
if baseurl[-1] != '/':
baseurl += '/'
if not baseurl.startswith('cooker'):
baseurl += "api/"
if branches:
baseurl += ";branch=%s" % ','.join(branches)
urls.append(baseurl)
return urls
# Set the default...
if args.branch:
branches = [args.branch]
else:
branches = (
self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES')
or 'master').split()
logger.debug('Trying branches: %s' % branches)
ignore_layers = []
if args.ignore:
ignore_layers.extend(args.ignore.split(','))
# Load the cooker DB
cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
cookerIndex.load_layerindex('cooker://', load='layerDependencies')
# Fast path, check if we already have what has been requested!
(dependencies,
invalidnames) = cookerIndex.find_dependencies(names=args.layername,
ignores=ignore_layers)
if not args.show_only and not invalidnames:
logger.plain("You already have the requested layer(s): %s" %
args.layername)
return 0
# The information to show is already in the cookerIndex
if invalidnames:
# General URL to use to access the layer index
# While there is ONE right now, we're expect users could enter several
apiurl = self.tinfoil.config_data.getVar(
'BBLAYERS_LAYERINDEX_URL').split()
if not apiurl:
logger.error("Cannot get BBLAYERS_LAYERINDEX_URL")
return 1
remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data)
for remoteurl in _construct_url(apiurl, branches):
logger.plain("Loading %s..." % remoteurl)
remoteIndex.load_layerindex(remoteurl)
if remoteIndex.is_empty():
logger.error("Remote layer index %s is empty for branches %s" %
(apiurl, branches))
return 1
lIndex = cookerIndex + remoteIndex
(dependencies,
invalidnames) = lIndex.find_dependencies(names=args.layername,
ignores=ignore_layers)
if invalidnames:
for invaluename in invalidnames:
logger.error('Layer "%s" not found in layer index' %
invaluename)
return 1
logger.plain("%s %s %s" %
("Layer".ljust(49), "Git repository (branch)".ljust(54),
"Subdirectory"))
logger.plain('=' * 125)
for deplayerbranch in dependencies:
layerBranch = dependencies[deplayerbranch][0]
# TODO: Determine display behavior
# This is the local content, uncomment to hide local
# layers from the display.
#if layerBranch.index.config['TYPE'] == 'cooker':
# continue
layerDeps = dependencies[deplayerbranch][1:]
requiredby = []
recommendedby = []
for dep in layerDeps:
if dep.required:
requiredby.append(dep.layer.name)
else:
recommendedby.append(dep.layer.name)
logger.plain(
'%s %s %s' %
(("%s:%s:%s" %
(layerBranch.index.config['DESCRIPTION'],
layerBranch.branch.name, layerBranch.layer.name)).ljust(50),
("%s (%s)" %
(layerBranch.layer.vcs_url, layerBranch.actual_branch)
).ljust(55), layerBranch.vcs_subdir))
if requiredby:
logger.plain(' required by: %s' % ' '.join(requiredby))
if recommendedby:
logger.plain(' recommended by: %s' % ' '.join(recommendedby))
if dependencies:
fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR')
if not fetchdir:
logger.error("Cannot get BBLAYERS_FETCH_DIR")
return 1
if not os.path.exists(fetchdir):
os.makedirs(fetchdir)
addlayers = []
for deplayerbranch in dependencies:
layerBranch = dependencies[deplayerbranch][0]
if layerBranch.index.config['TYPE'] == 'cooker':
# Anything loaded via cooker is already local, skip it
continue
subdir, name, layerdir = self.get_fetch_layer(
fetchdir, layerBranch.layer.vcs_url,
layerBranch.vcs_subdir, not args.show_only,
layerBranch.actual_branch, args.shallow)
if not name:
# Error already shown
return 1
addlayers.append((subdir, name, layerdir))
if not args.show_only:
localargs = argparse.Namespace()
localargs.layerdir = []
localargs.force = args.force
for subdir, name, layerdir in addlayers:
if os.path.exists(layerdir):
if subdir:
logger.plain(
"Adding layer \"%s\" (%s) to conf/bblayers.conf" %
(subdir, layerdir))
else:
logger.plain(
"Adding layer \"%s\" (%s) to conf/bblayers.conf" %
(name, layerdir))
localargs.layerdir.append(layerdir)
else:
break
if localargs.layerdir:
self.do_add_layer(localargs)
def update(self):
"""
Fetches layer, recipe and machine information from a layerindex
server
"""
os.system('setterm -cursor off')
self.apiurl = DEFAULT_LAYERINDEX_SERVER
if ToasterSetting.objects.filter(
name='CUSTOM_LAYERINDEX_SERVER').count() == 1:
self.apiurl = ToasterSetting.objects.get(
name='CUSTOM_LAYERINDEX_SERVER').value
assert self.apiurl is not None
# update branches; only those that we already have names listed in the
# Releases table
whitelist_branch_names = [
rel.branch_name for rel in Release.objects.all()
]
if len(whitelist_branch_names) == 0:
raise Exception("Failed to make list of branches to fetch")
logger.info("Fetching metadata for %s",
" ".join(whitelist_branch_names))
# We require a non-empty bb.data, but we can fake it with a dictionary
layerindex = layerindexlib.LayerIndex({"DUMMY": "VALUE"})
http_progress = Spinner()
http_progress.start()
if whitelist_branch_names:
url_branches = ";branch=%s" % ','.join(whitelist_branch_names)
else:
url_branches = ""
layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches))
http_progress.stop()
# We know we're only processing one entry, so we reference it here
# (this is cheating...)
index = layerindex.indexes[0]
# Map the layer index branches to toaster releases
li_branch_id_to_toaster_release = {}
logger.info("Processing releases")
total = len(index.branches)
for i, id in enumerate(index.branches):
li_branch_id_to_toaster_release[id] = \
Release.objects.get(name=index.branches[id].name)
self.mini_progress("Releases", i, total)
# keep a track of the layerindex (li) id mappings so that
# layer_versions can be created for these layers later on
li_layer_id_to_toaster_layer_id = {}
logger.info("Processing layers")
total = len(index.layerItems)
for i, id in enumerate(index.layerItems):
try:
l, created = Layer.objects.get_or_create(
name=index.layerItems[id].name)
l.up_date = index.layerItems[id].updated
l.summary = index.layerItems[id].summary
l.description = index.layerItems[id].description
if created:
# predefined layers in the fixtures (for example poky.xml)
# always preempt the Layer Index for these values
l.vcs_url = index.layerItems[id].vcs_url
l.vcs_web_url = index.layerItems[id].vcs_web_url
l.vcs_web_tree_base_url = index.layerItems[
id].vcs_web_tree_base_url
l.vcs_web_file_base_url = index.layerItems[
id].vcs_web_file_base_url
l.save()
except Layer.MultipleObjectsReturned:
logger.info("Skipped %s as we found multiple layers and "
"don't know which to update" %
index.layerItems[id].name)
li_layer_id_to_toaster_layer_id[id] = l.pk
self.mini_progress("layers", i, total)
# update layer_versions
logger.info("Processing layer versions")
# Map Layer index layer_branch object id to
# layer_version toaster object id
li_layer_branch_id_to_toaster_lv_id = {}
total = len(index.layerBranches)
for i, id in enumerate(index.layerBranches):
# release as defined by toaster map to layerindex branch
release = li_branch_id_to_toaster_release[
index.layerBranches[id].branch_id]
try:
lv, created = Layer_Version.objects.get_or_create(
layer=Layer.objects.get(pk=li_layer_id_to_toaster_layer_id[
index.layerBranches[id].layer_id]),
release=release)
except KeyError:
logger.warning(
"No such layerindex layer referenced by layerbranch %d" %
index.layerBranches[id].layer_id)
continue
if created:
lv.release = li_branch_id_to_toaster_release[
index.layerBranches[id].branch_id]
lv.up_date = index.layerBranches[id].updated
lv.commit = index.layerBranches[id].actual_branch
lv.dirpath = index.layerBranches[id].vcs_subdir
lv.save()
li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\
lv.pk
self.mini_progress("layer versions", i, total)
logger.info("Processing layer version dependencies")
dependlist = {}
for id in index.layerDependencies:
try:
lv = Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[
index.layerDependencies[id].layerbranch_id])
except Layer_Version.DoesNotExist as e:
continue
if lv not in dependlist:
dependlist[lv] = []
try:
layer_id = li_layer_id_to_toaster_layer_id[
index.layerDependencies[id].dependency_id]
dependlist[lv].append(
Layer_Version.objects.get(layer__pk=layer_id,
release=lv.release))
except Layer_Version.DoesNotExist:
logger.warning(
"Cannot find layer version (ls:%s),"
"up_id:%s lv:%s" %
(self, index.layerDependencies[id].dependency_id, lv))
total = len(dependlist)
for i, lv in enumerate(dependlist):
LayerVersionDependency.objects.filter(layer_version=lv).delete()
for lvd in dependlist[lv]:
LayerVersionDependency.objects.get_or_create(layer_version=lv,
depends_on=lvd)
self.mini_progress("Layer version dependencies", i, total)
# update Distros
logger.info("Processing distro information")
total = len(index.distros)
for i, id in enumerate(index.distros):
distro, created = Distro.objects.get_or_create(
name=index.distros[id].name,
layer_version=Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[
index.distros[id].layerbranch_id]))
distro.up_date = index.distros[id].updated
distro.name = index.distros[id].name
distro.description = index.distros[id].description
distro.save()
self.mini_progress("distros", i, total)
# update machines
logger.info("Processing machine information")
total = len(index.machines)
for i, id in enumerate(index.machines):
mo, created = Machine.objects.get_or_create(
name=index.machines[id].name,
layer_version=Layer_Version.objects.get(
pk=li_layer_branch_id_to_toaster_lv_id[
index.machines[id].layerbranch_id]))
mo.up_date = index.machines[id].updated
mo.name = index.machines[id].name
mo.description = index.machines[id].description
mo.save()
self.mini_progress("machines", i, total)
# update recipes; paginate by layer version / layer branch
logger.info("Processing recipe information")
total = len(index.recipes)
for i, id in enumerate(index.recipes):
try:
lv_id = li_layer_branch_id_to_toaster_lv_id[
index.recipes[id].layerbranch_id]
lv = Layer_Version.objects.get(pk=lv_id)
ro, created = Recipe.objects.get_or_create(
layer_version=lv, name=index.recipes[id].pn)
ro.layer_version = lv
ro.up_date = index.recipes[id].updated
ro.name = index.recipes[id].pn
ro.version = index.recipes[id].pv
ro.summary = index.recipes[id].summary
ro.description = index.recipes[id].description
ro.section = index.recipes[id].section
ro.license = index.recipes[id].license
ro.homepage = index.recipes[id].homepage
ro.bugtracker = index.recipes[id].bugtracker
ro.file_path = index.recipes[id].fullpath
ro.is_image = 'image' in index.recipes[id].inherits.split()
ro.save()
except Exception as e:
logger.warning("Failed saving recipe %s", e)
self.mini_progress("recipes", i, total)
os.system('setterm -cursor on')
def setUp(self):
LayersTest.setUp(self)
self.layerindex = layerindexlib.LayerIndex(self.d)
self.layerindex.load_layerindex(
'http://layers.openembedded.org/layerindex/api/;branch=sumo',
load=['layerDependencies'])
