def test_layerindex_store_split(self): self.layerindex.store_layerindex('file://%s' % self.tempdir, self.layerindex.indexes[0]) reload = layerindexlib.LayerIndex(self.d) reload.load_layerindex('file://%s' % self.tempdir) self.assertFalse(reload.is_empty(), msg="Layer index is empty") for type in self.layerindex.indexes[0]._index: if type == 'apilinks' or \ type == 'layerItems' or \ type in self.layerindex.indexes[0].config['local']: continue for id in getattr(self.layerindex.indexes[0], type): self.logger.debug("type %s" % (type)) self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number missing from reloaded data") self.logger.debug( "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id])) self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="reloaded data does not match original")
def test_layerindex_store_file(self): self.layerindex.store_layerindex('file://%s/file.json' % self.tempdir, self.layerindex.indexes[0]) self.assertTrue(os.path.isfile('%s/file.json' % self.tempdir), msg="Temporary file was not created by store_layerindex") reload = layerindexlib.LayerIndex(self.d) reload.load_layerindex('file://%s/file.json' % self.tempdir) self.assertFalse(reload.is_empty(), msg="Layerindex is empty") # Calculate layerItems in original index that should NOT be in reload layerItemNames = [] for itemId in self.layerindex.indexes[0].layerItems: layerItemNames.append(self.layerindex.indexes[0].layerItems[itemId].name) for layerBranchId in self.layerindex.indexes[0].layerBranches: layerItemNames.remove(self.layerindex.indexes[0].layerBranches[layerBranchId].layer.name) for itemId in reload.indexes[0].layerItems: self.assertFalse(reload.indexes[0].layerItems[itemId].name in layerItemNames, msg="Item reloaded when it shouldn't have been") # Compare the original to what we wrote... for type in self.layerindex.indexes[0]._index: if type == 'apilinks' or \ type == 'layerItems' or \ type in self.layerindex.indexes[0].config['local']: continue for id in getattr(self.layerindex.indexes[0], type): self.logger.debug(1, "type %s" % (type)) self.assertTrue(id in getattr(reload.indexes[0], type), msg="Id number not in reloaded index") self.logger.debug(1, "%s ? %s" % (getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id])) self.assertEqual(getattr(self.layerindex.indexes[0], type)[id], getattr(reload.indexes[0], type)[id], msg="Reloaded contents different")
def setUp(self): self.assertFalse( os.environ.get("BB_SKIP_NETTESTS") == "yes", msg="BB_SKIP_NETTESTS set, but we tried to test anyway") LayersTest.setUp(self) self.layerindex = layerindexlib.LayerIndex(self.d) self.layerindex.load_layerindex( 'http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])
def setUp(self): LayersTest.setUp(self) # Note this is NOT a comprehensive test of cooker, as we can't easily # configure the test data. But we can emulate the basics of the layer.conf # files, so that is what we will do. new_topdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "testdata") new_bbpath = os.path.join(new_topdir, "build") self.d.setVar('TOPDIR', new_topdir) self.d.setVar('BBPATH', new_bbpath) self.d = bb.parse.handle("%s/conf/bblayers.conf" % new_bbpath, self.d, True) for layer in self.d.getVar('BBLAYERS').split(): self.d = bb.parse.handle("%s/conf/layer.conf" % layer, self.d, True) self.layerindex = layerindexlib.LayerIndex(self.d) self.layerindex.load_layerindex('cooker://', load=['layerDependencies'])
def do_layerindex_fetch(self, args): """Fetches a layer from a layer index along with its dependent layers, and adds them to conf/bblayers.conf. """ def _construct_url(baseurls, branches): urls = [] for baseurl in baseurls: if baseurl[-1] != '/': baseurl += '/' if not baseurl.startswith('cooker'): baseurl += "api/" if branches: baseurl += ";branch=%s" % ','.join(branches) urls.append(baseurl) return urls # Set the default... if args.branch: branches = [args.branch] else: branches = ( self.tinfoil.config_data.getVar('LAYERSERIES_CORENAMES') or 'master').split() logger.debug('Trying branches: %s' % branches) ignore_layers = [] if args.ignore: ignore_layers.extend(args.ignore.split(',')) # Load the cooker DB cookerIndex = layerindexlib.LayerIndex(self.tinfoil.config_data) cookerIndex.load_layerindex('cooker://', load='layerDependencies') # Fast path, check if we already have what has been requested! (dependencies, invalidnames) = cookerIndex.find_dependencies(names=args.layername, ignores=ignore_layers) if not args.show_only and not invalidnames: logger.plain("You already have the requested layer(s): %s" % args.layername) return 0 # The information to show is already in the cookerIndex if invalidnames: # General URL to use to access the layer index # While there is ONE right now, we're expect users could enter several apiurl = self.tinfoil.config_data.getVar( 'BBLAYERS_LAYERINDEX_URL').split() if not apiurl: logger.error("Cannot get BBLAYERS_LAYERINDEX_URL") return 1 remoteIndex = layerindexlib.LayerIndex(self.tinfoil.config_data) for remoteurl in _construct_url(apiurl, branches): logger.plain("Loading %s..." % remoteurl) remoteIndex.load_layerindex(remoteurl) if remoteIndex.is_empty(): logger.error("Remote layer index %s is empty for branches %s" % (apiurl, branches)) return 1 lIndex = cookerIndex + remoteIndex (dependencies, invalidnames) = lIndex.find_dependencies(names=args.layername, ignores=ignore_layers) if invalidnames: for invaluename in invalidnames: logger.error('Layer "%s" not found in layer index' % invaluename) return 1 logger.plain("%s %s %s" % ("Layer".ljust(49), "Git repository (branch)".ljust(54), "Subdirectory")) logger.plain('=' * 125) for deplayerbranch in dependencies: layerBranch = dependencies[deplayerbranch][0] # TODO: Determine display behavior # This is the local content, uncomment to hide local # layers from the display. #if layerBranch.index.config['TYPE'] == 'cooker': # continue layerDeps = dependencies[deplayerbranch][1:] requiredby = [] recommendedby = [] for dep in layerDeps: if dep.required: requiredby.append(dep.layer.name) else: recommendedby.append(dep.layer.name) logger.plain( '%s %s %s' % (("%s:%s:%s" % (layerBranch.index.config['DESCRIPTION'], layerBranch.branch.name, layerBranch.layer.name)).ljust(50), ("%s (%s)" % (layerBranch.layer.vcs_url, layerBranch.actual_branch) ).ljust(55), layerBranch.vcs_subdir)) if requiredby: logger.plain(' required by: %s' % ' '.join(requiredby)) if recommendedby: logger.plain(' recommended by: %s' % ' '.join(recommendedby)) if dependencies: fetchdir = self.tinfoil.config_data.getVar('BBLAYERS_FETCH_DIR') if not fetchdir: logger.error("Cannot get BBLAYERS_FETCH_DIR") return 1 if not os.path.exists(fetchdir): os.makedirs(fetchdir) addlayers = [] for deplayerbranch in dependencies: layerBranch = dependencies[deplayerbranch][0] if layerBranch.index.config['TYPE'] == 'cooker': # Anything loaded via cooker is already local, skip it continue subdir, name, layerdir = self.get_fetch_layer( fetchdir, layerBranch.layer.vcs_url, layerBranch.vcs_subdir, not args.show_only, layerBranch.actual_branch, args.shallow) if not name: # Error already shown return 1 addlayers.append((subdir, name, layerdir)) if not args.show_only: localargs = argparse.Namespace() localargs.layerdir = [] localargs.force = args.force for subdir, name, layerdir in addlayers: if os.path.exists(layerdir): if subdir: logger.plain( "Adding layer \"%s\" (%s) to conf/bblayers.conf" % (subdir, layerdir)) else: logger.plain( "Adding layer \"%s\" (%s) to conf/bblayers.conf" % (name, layerdir)) localargs.layerdir.append(layerdir) else: break if localargs.layerdir: self.do_add_layer(localargs)
def update(self): """ Fetches layer, recipe and machine information from a layerindex server """ os.system('setterm -cursor off') self.apiurl = DEFAULT_LAYERINDEX_SERVER if ToasterSetting.objects.filter( name='CUSTOM_LAYERINDEX_SERVER').count() == 1: self.apiurl = ToasterSetting.objects.get( name='CUSTOM_LAYERINDEX_SERVER').value assert self.apiurl is not None # update branches; only those that we already have names listed in the # Releases table whitelist_branch_names = [ rel.branch_name for rel in Release.objects.all() ] if len(whitelist_branch_names) == 0: raise Exception("Failed to make list of branches to fetch") logger.info("Fetching metadata for %s", " ".join(whitelist_branch_names)) # We require a non-empty bb.data, but we can fake it with a dictionary layerindex = layerindexlib.LayerIndex({"DUMMY": "VALUE"}) http_progress = Spinner() http_progress.start() if whitelist_branch_names: url_branches = ";branch=%s" % ','.join(whitelist_branch_names) else: url_branches = "" layerindex.load_layerindex("%s%s" % (self.apiurl, url_branches)) http_progress.stop() # We know we're only processing one entry, so we reference it here # (this is cheating...) index = layerindex.indexes[0] # Map the layer index branches to toaster releases li_branch_id_to_toaster_release = {} logger.info("Processing releases") total = len(index.branches) for i, id in enumerate(index.branches): li_branch_id_to_toaster_release[id] = \ Release.objects.get(name=index.branches[id].name) self.mini_progress("Releases", i, total) # keep a track of the layerindex (li) id mappings so that # layer_versions can be created for these layers later on li_layer_id_to_toaster_layer_id = {} logger.info("Processing layers") total = len(index.layerItems) for i, id in enumerate(index.layerItems): try: l, created = Layer.objects.get_or_create( name=index.layerItems[id].name) l.up_date = index.layerItems[id].updated l.summary = index.layerItems[id].summary l.description = index.layerItems[id].description if created: # predefined layers in the fixtures (for example poky.xml) # always preempt the Layer Index for these values l.vcs_url = index.layerItems[id].vcs_url l.vcs_web_url = index.layerItems[id].vcs_web_url l.vcs_web_tree_base_url = index.layerItems[ id].vcs_web_tree_base_url l.vcs_web_file_base_url = index.layerItems[ id].vcs_web_file_base_url l.save() except Layer.MultipleObjectsReturned: logger.info("Skipped %s as we found multiple layers and " "don't know which to update" % index.layerItems[id].name) li_layer_id_to_toaster_layer_id[id] = l.pk self.mini_progress("layers", i, total) # update layer_versions logger.info("Processing layer versions") # Map Layer index layer_branch object id to # layer_version toaster object id li_layer_branch_id_to_toaster_lv_id = {} total = len(index.layerBranches) for i, id in enumerate(index.layerBranches): # release as defined by toaster map to layerindex branch release = li_branch_id_to_toaster_release[ index.layerBranches[id].branch_id] try: lv, created = Layer_Version.objects.get_or_create( layer=Layer.objects.get(pk=li_layer_id_to_toaster_layer_id[ index.layerBranches[id].layer_id]), release=release) except KeyError: logger.warning( "No such layerindex layer referenced by layerbranch %d" % index.layerBranches[id].layer_id) continue if created: lv.release = li_branch_id_to_toaster_release[ index.layerBranches[id].branch_id] lv.up_date = index.layerBranches[id].updated lv.commit = index.layerBranches[id].actual_branch lv.dirpath = index.layerBranches[id].vcs_subdir lv.save() li_layer_branch_id_to_toaster_lv_id[index.layerBranches[id].id] =\ lv.pk self.mini_progress("layer versions", i, total) logger.info("Processing layer version dependencies") dependlist = {} for id in index.layerDependencies: try: lv = Layer_Version.objects.get( pk=li_layer_branch_id_to_toaster_lv_id[ index.layerDependencies[id].layerbranch_id]) except Layer_Version.DoesNotExist as e: continue if lv not in dependlist: dependlist[lv] = [] try: layer_id = li_layer_id_to_toaster_layer_id[ index.layerDependencies[id].dependency_id] dependlist[lv].append( Layer_Version.objects.get(layer__pk=layer_id, release=lv.release)) except Layer_Version.DoesNotExist: logger.warning( "Cannot find layer version (ls:%s)," "up_id:%s lv:%s" % (self, index.layerDependencies[id].dependency_id, lv)) total = len(dependlist) for i, lv in enumerate(dependlist): LayerVersionDependency.objects.filter(layer_version=lv).delete() for lvd in dependlist[lv]: LayerVersionDependency.objects.get_or_create(layer_version=lv, depends_on=lvd) self.mini_progress("Layer version dependencies", i, total) # update Distros logger.info("Processing distro information") total = len(index.distros) for i, id in enumerate(index.distros): distro, created = Distro.objects.get_or_create( name=index.distros[id].name, layer_version=Layer_Version.objects.get( pk=li_layer_branch_id_to_toaster_lv_id[ index.distros[id].layerbranch_id])) distro.up_date = index.distros[id].updated distro.name = index.distros[id].name distro.description = index.distros[id].description distro.save() self.mini_progress("distros", i, total) # update machines logger.info("Processing machine information") total = len(index.machines) for i, id in enumerate(index.machines): mo, created = Machine.objects.get_or_create( name=index.machines[id].name, layer_version=Layer_Version.objects.get( pk=li_layer_branch_id_to_toaster_lv_id[ index.machines[id].layerbranch_id])) mo.up_date = index.machines[id].updated mo.name = index.machines[id].name mo.description = index.machines[id].description mo.save() self.mini_progress("machines", i, total) # update recipes; paginate by layer version / layer branch logger.info("Processing recipe information") total = len(index.recipes) for i, id in enumerate(index.recipes): try: lv_id = li_layer_branch_id_to_toaster_lv_id[ index.recipes[id].layerbranch_id] lv = Layer_Version.objects.get(pk=lv_id) ro, created = Recipe.objects.get_or_create( layer_version=lv, name=index.recipes[id].pn) ro.layer_version = lv ro.up_date = index.recipes[id].updated ro.name = index.recipes[id].pn ro.version = index.recipes[id].pv ro.summary = index.recipes[id].summary ro.description = index.recipes[id].description ro.section = index.recipes[id].section ro.license = index.recipes[id].license ro.homepage = index.recipes[id].homepage ro.bugtracker = index.recipes[id].bugtracker ro.file_path = index.recipes[id].fullpath ro.is_image = 'image' in index.recipes[id].inherits.split() ro.save() except Exception as e: logger.warning("Failed saving recipe %s", e) self.mini_progress("recipes", i, total) os.system('setterm -cursor on')
def setUp(self): LayersTest.setUp(self) self.layerindex = layerindexlib.LayerIndex(self.d) self.layerindex.load_layerindex( 'http://layers.openembedded.org/layerindex/api/;branch=sumo', load=['layerDependencies'])