def _load_contents_data(self, arch_name, component): contents_basename = "Contents-%s.gz" % (arch_name) contents_fname = os.path.join(self._mirror_dir, "dists", self._suite_name, component, contents_basename) # Ubuntu does not place the Contents file in a component-specific directory, # so fall back to the global one. if not os.path.isfile(contents_fname): path = os.path.join(self._mirror_dir, "dists", self._suite_name, contents_basename) if os.path.isfile(path): contents_fname = path # load and preprocess the large file. # we don't show mercy to memory here, we just want the icon lookup to be fast, # so we need to cache the data. f = gzip.open(contents_fname, 'r') for line in f: line = _decode_contents_line(line) if line.startswith("usr/share/icons/hicolor/") or line.startswith("usr/share/pixmaps/"): self._contents_data.append(line) continue # allow Oxygen icon theme, needed to support KDE apps if line.startswith("usr/share/icons/oxygen"): self._contents_data.append(line) continue # in rare events, GNOME needs the same treatment, so special-case Adwaita as well if line.startswith("usr/share/icons/Adwaita"): self._contents_data.append(line) continue f.close() new_pkgs = read_packages_dict_from_file(self._mirror_dir, self._suite_name, component, arch_name) self._packages_dict.update(new_pkgs)
def update_html(self): dep11_hintsdir = os.path.join(self._export_dir, "hints") if not os.path.exists(dep11_hintsdir): return dep11_minfodir = os.path.join(self._export_dir, "data") if not os.path.exists(dep11_minfodir): return export_dir = self._html_export_dir media_dir = os.path.join(self._export_dir, "media") noimage_url = os.path.join(self._html_url, "static", "img", "no-image.png") # Render archive suites index page self.render_template("suites_index.html", export_dir, "index.html", suites=self._suites_data.keys()) # TODO: Remove old HTML files for suite_name in self._suites_data: suite = self._suites_data[suite_name] export_dir = os.path.join(self._export_dir, "html", suite_name) suite_error_count = 0 suite_warning_count = 0 suite_info_count = 0 suite_metainfo_count = 0 for component in suite['components']: issue_summaries = dict() mdata_summaries = dict() export_dir_section = os.path.join(self._export_dir, "html", suite_name, component) export_dir_issues = os.path.join(export_dir_section, "issues") export_dir_metainfo = os.path.join(export_dir_section, "metainfo") error_count = 0 warning_count = 0 info_count = 0 metainfo_count = 0 hint_pages = dict() cpt_pages = dict() for arch in suite['architectures']: h_fname = os.path.join(dep11_hintsdir, suite_name, component, "DEP11Hints_%s.yml.gz" % (arch)) hints_data = None if os.path.isfile(h_fname): f = gzip.open(h_fname, 'r') hints_data = yaml.safe_load_all(f.read()) f.close() d_fname = os.path.join(dep11_minfodir, suite_name, component, "Components-%s.yml.gz" % (arch)) dep11_data = None if os.path.isfile(d_fname): f = gzip.open(d_fname, 'r') dep11_data = yaml.safe_load_all(f.read()) f.close() pkg_index = read_packages_dict_from_file(self._archive_root, suite_name, component, arch) if hints_data: for hdata in hints_data: pkg_name = hdata['Package'] pkg_id = hdata.get('PackageID') if not pkg_id: pkg_id = pkg_name pkg = pkg_index.get(pkg_name) maintainer = None if pkg: maintainer = pkg['maintainer'] if not maintainer: maintainer = "Unknown" if not issue_summaries.get(maintainer): issue_summaries[maintainer] = dict() hints_raw = hdata.get('Hints', list()) # expand all hints to show long descriptions errors = list() warnings = list() infos = list() for hint in hints_raw: ehint = self._expand_hint(hint) severity = ehint['severity'] if severity == "info": infos.append(ehint) elif severity == "warning": warnings.append(ehint) else: errors.append(ehint) if not hint_pages.get(pkg_name): hint_pages[pkg_name] = list() # we fold multiple architectures with the same issues into one view pkid_noarch = pkg_id if "/" in pkg_id: pkid_noarch = pkg_id[:pkg_id.rfind("/")] pcid = "" if hdata.get('ID'): pcid = "%s: %s" % (pkid_noarch, hdata.get('ID')) else: pcid = pkid_noarch page_data = {'identifier': pcid, 'errors': errors, 'warnings': warnings, 'infos': infos, 'archs': [arch]} try: l = hint_pages[pkg_name] index = next(i for i, v in enumerate(l) if equal_dicts(v, page_data, ['archs'])) hint_pages[pkg_name][index]['archs'].append(arch) except StopIteration: hint_pages[pkg_name].append(page_data) # add info to global issue count error_count += len(errors) warning_count += len(warnings) info_count += len(infos) # add info for global index if not issue_summaries[maintainer].get(pkg_name): issue_summaries[maintainer][pkg_name] = {'error_count': len(errors), 'warning_count': len(warnings), 'info_count': len(infos)} if dep11_data: for mdata in dep11_data: pkg_name = mdata.get('Package') if not pkg_name: # we probably hit the header continue pkg = pkg_index.get(pkg_name) maintainer = None if pkg: maintainer = pkg['maintainer'] if not maintainer: maintainer = "Unknown" if not mdata_summaries.get(maintainer): mdata_summaries[maintainer] = dict() # ugly hack to have the screenshot entries linked #if mdata.get('Screenshots'): # sshot_baseurl = os.path.join(self._dep11_url, component) # for i in range(len(mdata['Screenshots'])): # url = mdata['Screenshots'][i]['source-image']['url'] # url = "<a href=\"%s\">%s</a>" % (os.path.join(sshot_baseurl, url), url) # mdata['Screenshots'][i]['source-image']['url'] = Markup(url) # thumbnails = mdata['Screenshots'][i]['thumbnails'] # for j in range(len(thumbnails)): # url = thumbnails[j]['url'] # url = "<a href=\"%s\">%s</a>" % (os.path.join(sshot_baseurl, url), url) # thumbnails[j]['url'] = Markup(url) # mdata['Screenshots'][i]['thumbnails'] = thumbnails mdata_yml = dict_to_dep11_yaml(mdata) mdata_yml = self._highlight_yaml(mdata_yml) cid = mdata.get('ID') # try to find an icon for this component (if it's a GUI app) icon_url = None if mdata['Type'] == 'desktop-app' or mdata['Type'] == "web-app": icon_name = mdata['Icon'].get("cached") cptgid = build_cpt_global_id(cid, mdata.get('X-Source-Checksum')) if icon_name and cptgid: icon_fname = os.path.join(component, cptgid, "icons", "64x64", icon_name) if os.path.isfile(os.path.join(media_dir, icon_fname)): icon_url = os.path.join(self._dep11_url, icon_fname) else: icon_url = noimage_url else: icon_url = noimage_url else: icon_url = os.path.join(self._html_url, "static", "img", "cpt-nogui.png") if not cpt_pages.get(pkg_name): cpt_pages[pkg_name] = list() page_data = {'cid': cid, 'mdata': mdata_yml, 'icon_url': icon_url, 'archs': [arch]} try: l = cpt_pages[pkg_name] index = next(i for i, v in enumerate(l) if equal_dicts(v, page_data, ['archs'])) cpt_pages[pkg_name][index]['archs'].append(arch) except StopIteration: cpt_pages[pkg_name].append(page_data) # increase valid metainfo count metainfo_count += 1 # check if we had this package, and add to summary pksum = mdata_summaries[maintainer].get(pkg_name) if not pksum: pksum = dict() if pksum.get('cids'): if not cid in pksum['cids']: pksum['cids'].append(cid) else: pksum['cids'] = [cid] mdata_summaries[maintainer][pkg_name] = pksum if not dep11_data and not hints_data: log.warning("Suite %s/%s/%s does not contain DEP-11 data or issue hints.", suite_name, component, arch) # now write the HTML pages with the previously collected & transformed issue data for pkg_name, entry_list in hint_pages.items(): # render issues page self.render_template("issues_page.html", export_dir_issues, "%s.html" % (pkg_name), package_name=pkg_name, entries=entry_list, suite=suite_name, section=component) # render page with all components found in a package for pkg_name, cptlist in cpt_pages.items(): # render metainfo page self.render_template("metainfo_page.html", export_dir_metainfo, "%s.html" % (pkg_name), package_name=pkg_name, cpts=cptlist, suite=suite_name, section=component) # Now render our issue index page self.render_template("issues_index.html", export_dir_issues, "index.html", package_summaries=issue_summaries, suite=suite_name, section=component) # ... and the metainfo index page self.render_template("metainfo_index.html", export_dir_metainfo, "index.html", package_summaries=mdata_summaries, suite=suite_name, section=component) validate_result = "Validation was not performed." if dep11_data: # do format validation validator = DEP11Validator() ret = validator.validate_file(d_fname) if ret: validate_result = "No errors found." else: validate_result = "" for issue in validator.issue_list: validate_result += issue.replace("FATAL", "<strong>FATAL</strong>")+"<br/>\n" # sum up counts for suite statistics suite_metainfo_count += metainfo_count suite_error_count += error_count suite_warning_count += warning_count suite_info_count += info_count # calculate statistics for this component count = metainfo_count + error_count + warning_count + info_count valid_perc = 100/count*metainfo_count if count > 0 else 0 error_perc = 100/count*error_count if count > 0 else 0 warning_perc = 100/count*warning_count if count > 0 else 0 info_perc = 100/count*info_count if count > 0 else 0 # Render our overview page self.render_template("section_overview.html", export_dir_section, "index.html", suite=suite_name, section=component, valid_percentage=valid_perc, error_percentage=error_perc, warning_percentage=warning_perc, info_percentage=info_perc, metainfo_count=metainfo_count, error_count=error_count, warning_count=warning_count, info_count=info_count, validate_result=validate_result) # calculate statistics for this suite count = suite_metainfo_count + suite_error_count + suite_warning_count + suite_info_count valid_perc = 100/count*suite_metainfo_count if count > 0 else 0 error_perc = 100/count*suite_error_count if count > 0 else 0 warning_perc = 100/count*suite_warning_count if count > 0 else 0 info_perc = 100/count*suite_info_count if count > 0 else 0 # Render archive components index/overview page self.render_template("sections_index.html", export_dir, "index.html", sections=suite['components'], suite=suite_name, valid_percentage=valid_perc, error_percentage=error_perc, warning_percentage=warning_perc, info_percentage=info_perc, metainfo_count=suite_metainfo_count, error_count=suite_error_count, warning_count=suite_warning_count, info_count=suite_info_count) # Copy the static files target_static_dir = os.path.join(self._export_dir, "html", "static") shutil.rmtree(target_static_dir, ignore_errors=True) shutil.copytree(os.path.join(self._template_dir, "static"), target_static_dir)
def _get_packages_for(self, suite, component, arch): return read_packages_dict_from_file(self._archive_root, suite, component, arch).values()