def insert_item(self, data, src, target, pedigree, contentsource): """Queue item to be inserted in subsequent call to insert_version This adds the item to self.inserts which is then handled in insert_version. That allows the code to have context on all the items for a given version, and "choose" one. Ie, if both root.tar.xz and squashfs are available, preference can be given to the root.tar.gz. """ product_name, version_name, item_name = pedigree if product_name not in self.inserts: self.inserts[product_name] = {} if version_name not in self.inserts[product_name]: self.inserts[product_name][version_name] = {} if 'ftype' in data: ftype = data['ftype'] else: flat = util.products_exdata(src, pedigree, include_top=False) ftype = flat.get('ftype') self.inserts[product_name][version_name][item_name] = (ftype, (data, src, target, pedigree, contentsource))
def get_product_items(self, itemfilter=None): """ Parse products from this ContentSource, matching the filter. :type itemfilter: Optional[SSFilter] """ itemfilter = itemfilter or AndFilter() # empty AndFilter is true contents = super().read() super().close() stream = json.loads(contents) assert stream.get('format') == 'products:1.0', \ 'simplestreams product stream is of supported version' expand_tree(stream) for product_name, product in stream.get('products', {}).items(): for version_name, version in product.get('versions', {}).items(): for item_name, item in version.get('items', {}).items(): pedigree = (product_name, version_name, item_name) item = products_exdata(stream, pedigree) item = self._extend_item_info(item) if itemfilter.matches(item): yield item
def insert_item(self, data, src, target, pedigree, contentsource): """Overridable from `BasicMirrorWriter`.""" item = products_exdata(src, pedigree) checksums = item_checksums(data) tag = checksums["sha256"] size = data["size"] ftype = item["ftype"] filename = os.path.basename(item["path"]) if ftype == "archive.tar.xz": links = extract_archive_tar( self.store, filename, tag, checksums, size, contentsource ) else: links = insert_file( self.store, filename, tag, checksums, size, contentsource ) osystem = get_os_from_product(item) # link_resources creates a hardlink for every subarch. Every Ubuntu # product in a SimpleStream contains a list of subarches which list # what subarches are a subset of that subarch. For example Xenial # ga-16.04 has the subarches list hwe-{p,q,r,s,t,u,v,w},ga-16.04. # Kernel flavors are the same arch, the only difference is the kernel # config. So ga-16.04-lowlatency has the same subarch list as ga-16.04. # If we create hard links for all subarches a kernel flavor may # overwrite the generic kernel hard link. This happens if a kernel # flavor is processed after the generic kernel. Since MAAS doesn't use # the other hard links only create hard links for the subarch of the # product we have and a rolling link if it's a rolling kernel. if "subarch" in item: # MAAS uses the 'generic' subarch when it doesn't know which # subarch to use. This happens during enlistment and commissioning. # Allow the 'generic' kflavor to own the 'generic' hardlink. The # generic kernel should always be the ga kernel for xenial+, # hwe-<first letter of release> for older releases. if item.get("kflavor") == "generic" and ( item["subarch"].startswith("ga-") or item["subarch"] == "hwe-%s" % item["release"][0] ): subarches = {item["subarch"], "generic"} else: subarches = {item["subarch"]} else: subarches = {"generic"} if item.get("rolling", False): subarch_parts = item["subarch"].split("-") subarch_parts[1] = "rolling" subarches.add("-".join(subarch_parts)) link_resources( snapshot_path=self.root_path, links=links, osystem=osystem, arch=item["arch"], release=item["release"], label=item["label"], subarches=subarches, bootloader_type=item.get("bootloader-type"), )
def insert_item(self, data, src, target, pedigree, contentsource): # src and target are top level products:1.0 # data is src['products'][ped[0]]['versions'][ped[1]]['items'][ped[2]] # contentsource is a ContentSource if 'path' exists in data or None data = s_util.products_exdata(src, pedigree) if 'path' in data: data.update({'item_url': contentsource.url}) self.json_entries.append(data)
def insert_item(self, data, src, target, pedigree, contentsource): """See `ObjectStoreMirrorWriter`.""" super(MAASMirrorWriter, self).insert_item( data, src, target, pedigree, contentsource) path = data.get('path', None) flat = util.products_exdata(src, pedigree) if path is not None: self.extract_item(path, flat)
def _moditem(src, path, pedigree, modfunc): # load the products data at 'path' in 'src' mirror, then call modfunc # on the data found at pedigree. and store the updated data. sobj = src.objectstore tree = util.load_content(sobj.source(path).read()) item = util.products_exdata(tree, pedigree, insert_fieldnames=False) util.products_set(tree, modfunc(item), pedigree) sobj.insert_content(path, util.dump_data(tree))
def insert_item(self, data, src, target, pedigree, contentsource): """Overridable from `BasicMirrorWriter`.""" item = products_exdata(src, pedigree) if self.validate_products and not validate_product(item, pedigree[0]): maaslog.warning("Ignoring unsupported product %s" % pedigree[0]) return os = get_os_from_product(item) arch = item["arch"] subarches = item.get("subarches", "generic") if item.get("bootloader-type") is None: release = item["release"] kflavor = item.get("kflavor", "generic") else: release = item["bootloader-type"] kflavor = "bootloader" label = item["label"] base_image = ImageSpec(os, arch, None, kflavor, release, label) compact_item = clean_up_repo_item(item) if os == "ubuntu-core": # For Ubuntu Core we only want one entry per release/arch/gadget gadget = item.get("gadget_snap", "generic") kflavor = item.get("kernel_snap", "generic") release = "%s-%s" % (release, gadget) self.boot_images_dict.setdefault( base_image._replace( subarch="generic", kflavor=kflavor, release=release ), compact_item, ) else: for subarch in subarches.split(","): self.boot_images_dict.setdefault( base_image._replace(subarch=subarch), compact_item ) # HWE resources need to map to a specfic resource, and not just to # any of the supported subarchitectures for that resource. subarch = item.get("subarch", "generic") self.boot_images_dict.set( base_image._replace(subarch=subarch), compact_item ) if os == "ubuntu" and item.get("version") is not None: # HWE resources with generic, should map to the HWE that ships # with that release. Starting with Xenial kernels changed from # using the naming format hwe-<letter> to ga-<version>. Look # for both. hwe_archs = ["ga-%s" % item["version"], "hwe-%s" % release[0]] if subarch in hwe_archs and "generic" in subarches: self.boot_images_dict.set( base_image._replace(subarch="generic"), compact_item )
def insert_item(self, data, src, target, pedigree, contentsource): """Overridable from `BasicMirrorWriter`.""" item = products_exdata(src, pedigree) checksums = item_checksums(data) tag = checksums['sha256'] size = data['size'] ftype = item['ftype'] filename = os.path.basename(item['path']) if ftype == 'archive.tar.xz': links = extract_archive_tar( self.store, filename, tag, checksums, size, contentsource) elif ftype == 'root-image.gz': links = insert_root_image( self.store, tag, checksums, size, contentsource) else: links = insert_file( self.store, filename, tag, checksums, size, contentsource) osystem = get_os_from_product(item) # link_resources creates a hardlink for every subarch. Every Ubuntu # product in a SimpleStream contains a list of subarches which list # what subarches are a subset of that subarch. For example Xenial # ga-16.04 has the subarches list hwe-{p,q,r,s,t,u,v,w},ga-16.04. # Kernel flavors are the same arch, the only difference is the kernel # config. So ga-16.04-lowlatency has the same subarch list as ga-16.04. # If we create hard links for all subarches a kernel flavor may # overwrite the generic kernel hard link. This happens if a kernel # flavor is processed after the generic kernel. Since MAAS doesn't use # the other hard links only create hard links for the subarch of the # product we have and a rolling link if it's a rolling kernel. if 'subarch' in item: # MAAS uses the 'generic' subarch when it doesn't know which # subarch to use. This happens during enlistment and commissioning. # Allow the 'generic' kflavor to own the 'generic' hardlink. if item.get('kflavor') == 'generic': subarches = {item['subarch'], 'generic'} else: subarches = {item['subarch']} else: subarches = {'generic'} if item.get('rolling', False): subarch_parts = item['subarch'].split('-') subarch_parts[1] = 'rolling' subarches.add('-'.join(subarch_parts)) link_resources( snapshot_path=self.root_path, links=links, osystem=osystem, arch=item['arch'], release=item['release'], label=item['label'], subarches=subarches, bootloader_type=item.get('bootloader-type'))
def remove_item(self, data, src, target, pedigree): """See `ObjectStoreMirrorWriter`. Remove items from our local mirror that are no longer available upstream. """ if not self.delete: # Caller didn't ask for obsolete items to be deleted. return super(MAASMirrorWriter, self).remove_item(data, src, target, pedigree) metadata = util.products_exdata(src, pedigree) name = get_target_name(**metadata) tgt_admin_delete(name) remove(get_conf_path(self.local_path, name)) shutil.rmtree(self._target_dir(metadata))
def insert_item(self, data, src, target, pedigree, contentsource): """Insert item received. src and target are top level products:1.0 data is src['products'][ped[0]]['versions'][ped[1]]['items'][ped[2]] contentsource is a ContentSource if 'path' exists in data or None Args: data: Data from simplestreams src: TBD target: TBD pedigree: TBD contentsource: TBD """ data = s_util.products_exdata(src, pedigree) if 'path' in data: data.update({'item_url': contentsource.url}) self.json_entries.append(data)
def insert_item(self, data, src, target, pedigree, contentsource): mdata = util.products_exdata(src, pedigree) tmp_path = None tmp_del = None extra = {} if 'path' in data: extra.update({'item_url': contentsource.url}) if not self.config.get('item_skip_download', False): try: (tmp_path, tmp_del) = util.get_local_copy(contentsource) extra['path_local'] = tmp_path finally: contentsource.close() try: ret = self.call_hook('insert_item', data=mdata, extra=extra) finally: if tmp_del and os.path.exists(tmp_path): os.unlink(tmp_path) return ret
def query_ptree(ptree, max_num=None, ifilters=None, path2url=None): results = [] pkey = 'products' verkey = 'versions' for prodname, proddata in sorted(ptree.get(pkey, {}).items()): if verkey not in proddata: continue cur = 0 for vername in sorted(proddata[verkey].keys(), reverse=True): if max_num is not None and cur >= max_num: break verdata = proddata[verkey][vername] cur += 1 for itemname, itemdata in sorted(verdata.get('items', {}).items()): flat = sutil.products_exdata(ptree, (prodname, vername, itemname)) if ifilters is not None and len(ifilters) > 0: if not filters.filter_dict(ifilters, flat): continue if path2url and 'path' in flat: flat['item_url'] = path2url(flat['path']) results.append(flat) return results
def load_products(self, path=None, content_id=None): """ Load metadata for all currently uploaded active images in Glance. Uses glance as the definitive store, but loads metadata from existing simplestreams indexes as well. """ my_cid = self.content_id # glance is the definitive store. Any data loaded from the store # is secondary. store_t = None if self.store: try: path = self._cidpath(my_cid) store_t = util.load_content(self.store.source(path).read()) except IOError as e: if e.errno != errno.ENOENT: raise if not store_t: store_t = empty_iid_products(my_cid) glance_t = empty_iid_products(my_cid) images = self.gclient.images.list() for image in images: if self.glance_api_version == "1": image = image.to_dict() props = image['properties'] else: props = copy.deepcopy(image) if image['owner'] != self.tenant_id: continue if props.get('content_id') != my_cid: continue if image.get('status') != "active": LOG.warn("Ignoring inactive image %s with status '%s'" % (image['id'], image.get('status'))) continue source_content_id = props.get('source_content_id') product = props.get('product_name') version = props.get('version_name') item = props.get('item_name') if not (version and product and item and source_content_id): LOG.warn("%s missing required fields" % image['id']) continue # get data from the datastore for this item, if it exists # and then update that with glance data (just in case different) try: item_data = util.products_exdata(store_t, ( product, version, item, ), include_top=False, insert_fieldnames=False) except KeyError: item_data = {} # If original simplestreams-metadata is stored on the image, # use that as well. if 'simplestreams_metadata' in props: simplestreams_metadata = json.loads( props.get('simplestreams_metadata')) else: simplestreams_metadata = {} item_data.update(simplestreams_metadata) item_data.update({'name': image['name'], 'id': image['id']}) if 'owner_id' not in item_data: item_data['owner_id'] = self.tenant_id util.products_set(glance_t, item_data, ( product, version, item, )) for product in glance_t['products']: glance_t['products'][product]['region'] = self.region glance_t['products'][product]['endpoint'] = self.auth_url return glance_t
def filter_item(filters, data, src, pedigree): "Apply filter list to a products entity. Flatten before doing so." return filter_dict(filters, util.products_exdata(src, pedigree))
def insert_item(self, data, src, target, pedigree, contentsource): data = util.products_exdata(src, pedigree) self.items.append(dict_to_item(data))
def remove_version(self, data, src, target, pedigree): return self.call_hook('remove_version', data=util.products_exdata(src, pedigree))
def remove_item(self, data, src, target, pedigree): return self.call_hook('remove_item', data=util.products_exdata(target, pedigree))
def sync_products(self, reader, path=None, src=None, content=None): (src, content) = _get_data_content(path, src, content, reader) util.expand_tree(src) check_tree_paths(src) content_id = src['content_id'] target = self.load_products(path, content_id) if not target: target = util.stringitems(src) util.expand_tree(target) stree = src.get('products', {}) if 'products' not in target: target['products'] = {} tproducts = target['products'] filtered_products = [] prodname = None # Apply filters to items before filtering versions for prodname, product in list(stree.items()): for vername, version in list(product.get('versions', {}).items()): for itemname, item in list(version.get('items', {}).items()): pgree = (prodname, vername, itemname) if not self.filter_item(item, src, target, pgree): LOG.debug("Filtered out item: %s/%s", itemname, item) del stree[prodname]['versions'][vername]['items'][ itemname] if not stree[prodname]['versions'][vername].get( 'items', {}): del stree[prodname]['versions'][vername] if not stree[prodname].get('versions', {}): del stree[prodname] for prodname, product in stree.items(): if not self.filter_product(product, src, target, (prodname, )): filtered_products.append(prodname) continue if prodname not in tproducts: tproducts[prodname] = util.stringitems(product) tproduct = tproducts[prodname] if 'versions' not in tproduct: tproduct['versions'] = {} src_filtered_items = [] def _filter(itemkey): ret = self.filter_version(product['versions'][itemkey], src, target, (prodname, itemkey)) if not ret: src_filtered_items.append(itemkey) return ret (to_add, to_remove) = util.resolve_work( src=list(product.get('versions', {}).keys()), target=list(tproduct.get('versions', {}).keys()), maxnum=self.config.get('max_items'), keep=self.config.get('keep_items'), itemfilter=_filter) LOG.info("%s/%s: to_add=%s to_remove=%s", content_id, prodname, to_add, to_remove) tversions = tproduct['versions'] skipped_versions = [] for vername in to_add: version = product['versions'][vername] if vername not in tversions: tversions[vername] = util.stringitems(version) added_items = [] for itemname, item in version.get('items', {}).items(): pgree = (prodname, vername, itemname) added_items.append(itemname) ipath = item.get('path', None) ipath_cs = None if ipath and reader: if self.checksumming_reader: flat = util.products_exdata(src, pgree) ipath_cs = cs.ChecksummingContentSource( csrc=reader.source(ipath), size=flat.get('size'), checksums=checksum_util.item_checksums(flat)) else: ipath_cs = reader.source(ipath) self.insert_item(item, src, target, pgree, ipath_cs) if len(added_items): # do not insert versions that had all items filtered self.insert_version(version, src, target, (prodname, vername)) else: skipped_versions.append(vername) for vername in skipped_versions: if vername in tproduct['versions']: del tproduct['versions'][vername] if self.config.get('delete_filtered_items', False): tkeys = tproduct.get('versions', {}).keys() for v in src_filtered_items: if v not in to_remove and v in tkeys: to_remove.append(v) LOG.info("After deletions %s/%s: to_add=%s to_remove=%s", content_id, prodname, to_add, to_remove) for vername in to_remove: tversion = tversions[vername] for itemname in list(tversion.get('items', {}).keys()): self.remove_item(tversion['items'][itemname], src, target, (prodname, vername, itemname)) self.remove_version(tversion, src, target, (prodname, vername)) del tversions[vername] self.insert_product(tproduct, src, target, (prodname, )) # FIXME: below will remove products if they're in target # (result of load_products) but not in the source products. # that could accidentally delete a lot. # del_products = [] if self.config.get('delete_products', False): del_products.extend( [p for p in list(tproducts.keys()) if p not in stree]) if self.config.get('delete_filtered_products', False): del_products.extend( [p for p in filtered_products if p not in stree]) for prodname in del_products: # FIXME: we remove a product here, but unless that acts # recursively, nothing will remove the items in that product self.remove_product(tproducts[prodname], src, target, (prodname, )) del tproducts[prodname] self.insert_products(path, target, content)
def insert_product(self, data, src, target, pedigree): return self.call_hook('insert_product', data=util.products_exdata(src, pedigree))
def _call_filter(self, name, src, pedigree): data = util.products_exdata(src, pedigree) (ret, _output) = self.call_hook(name, data=data, rcs=[0, 1]) return ret == 0
def _insert_item(self, data, src, target, pedigree, contentsource): """ Upload image into glance and add image metadata to simplestreams index. `data` is the metadata for a particular image file from the source: unused since all that data is present in the `src` entry for the corresponding image as well. `src` contains the entire simplestreams index from the image syncing source. `target` is the simplestreams index for currently available images in glance (generated by load_products()) to add this item to. `pedigree` is a "path" to get to the `data` for the image we desire, a tuple of (product_name, version_name, image_type). `contentsource` is a ContentSource to download the actual image data from. """ # Extract and flatten metadata for a product image matching # (product-name, version-name, image-type) # from the tuple `pedigree` in the source simplestreams index. flattened_img_data = util.products_exdata(src, pedigree, include_top=False) tmp_path = None full_image_name = "{}{}".format( self.name_prefix, flattened_img_data.get('pubname', flattened_img_data.get('name'))) if not full_image_name.endswith(flattened_img_data['item_name']): full_image_name += "-{}".format(flattened_img_data['item_name']) # Download images locally into a temporary file. tmp_path, new_size, new_md5 = self.download_image( contentsource, flattened_img_data) hypervisor_mapping = self.config.get('hypervisor_mapping', False) glance_props = self.create_glance_properties(target['content_id'], src['content_id'], flattened_img_data, hypervisor_mapping) create_kwargs = self.prepare_glance_arguments(full_image_name, flattened_img_data, new_md5, new_size, glance_props) target_sstream_item = self.adapt_source_entry(flattened_img_data, hypervisor_mapping, full_image_name, new_md5, new_size) try: if self.glance_api_version == "1": # Set data as string if v1 create_kwargs['data'] = open(tmp_path, 'rb') else: # Keep properties for v2 update call _properties = create_kwargs['properties'] del create_kwargs['properties'] glance_image = self.gclient.images.create(**create_kwargs) target_sstream_item['id'] = glance_image.id if self.glance_api_version == "2": # Upload for v2 self.gclient.images.upload(glance_image.id, open(tmp_path, 'rb')) # Update properties for v2 self.gclient.images.update(glance_image.id, **_properties) # Validate the image checksum and size. This will throw an # IOError if they do not match. self.validate_image(glance_image.id, new_md5, new_size) print("created %s: %s" % (glance_image.id, full_image_name)) finally: if tmp_path and os.path.exists(tmp_path): os.unlink(tmp_path) util.products_set(target, target_sstream_item, pedigree) # We can safely ignore path and content arguments since they are # unused in insert_products below. self.insert_products(None, target, None)
def filter_version(self, data, src, target, pedigree): """Overridable from `BasicMirrorWriter`.""" return self.product_mapping.contains(products_exdata(src, pedigree))
def insert_item(self, data, src, target, pedigree, contentsource): data = util.products_exdata(src, pedigree) if 'size' in data and 'path' in data: self.downloading.append( (pedigree, data['path'], int(data['size'])))
def insert_item(self, data, src, target, pedigree, contentsource): data = util.products_exdata(src, pedigree) if 'size' in data and 'path' in data and 'pubname' in data: self.items[data['pubname']] = int(data['size'])
def remove_item(self, data, src, target, pedigree): data = util.products_exdata(src, pedigree) if 'size' in data and 'path' in data: self.removing.append((pedigree, data['path'], int(data['size'])))