def _load_contents_data(self, arch_name, component):
contents_basename = "Contents-%s.gz" % (arch_name)
contents_fname = os.path.join(self._mirror_dir, "dists", self._suite_name, component, contents_basename)
# Ubuntu does not place the Contents file in a component-specific directory,
# so fall back to the global one.
if not os.path.isfile(contents_fname):
path = os.path.join(self._mirror_dir, "dists", self._suite_name, contents_basename)
if os.path.isfile(path):
contents_fname = path
# load and preprocess the large file.
# we don't show mercy to memory here, we just want the icon lookup to be fast,
# so we need to cache the data.
f = gzip.open(contents_fname, 'r')
for line in f:
line = _decode_contents_line(line)
if line.startswith("usr/share/icons/hicolor/") or line.startswith("usr/share/pixmaps/"):
self._contents_data.append(line)
continue
# allow Oxygen icon theme, needed to support KDE apps
if line.startswith("usr/share/icons/oxygen"):
self._contents_data.append(line)
continue
# in rare events, GNOME needs the same treatment, so special-case Adwaita as well
if line.startswith("usr/share/icons/Adwaita"):
self._contents_data.append(line)
continue
f.close()
new_pkgs = read_packages_dict_from_file(self._mirror_dir, self._suite_name, component, arch_name)
self._packages_dict.update(new_pkgs)
def update_html(self):
dep11_hintsdir = os.path.join(self._export_dir, "hints")
if not os.path.exists(dep11_hintsdir):
return
dep11_minfodir = os.path.join(self._export_dir, "data")
if not os.path.exists(dep11_minfodir):
return
export_dir = self._html_export_dir
media_dir = os.path.join(self._export_dir, "media")
noimage_url = os.path.join(self._html_url, "static", "img", "no-image.png")
# Render archive suites index page
self.render_template("suites_index.html", export_dir, "index.html", suites=self._suites_data.keys())
# TODO: Remove old HTML files
for suite_name in self._suites_data:
suite = self._suites_data[suite_name]
export_dir = os.path.join(self._export_dir, "html", suite_name)
suite_error_count = 0
suite_warning_count = 0
suite_info_count = 0
suite_metainfo_count = 0
for component in suite['components']:
issue_summaries = dict()
mdata_summaries = dict()
export_dir_section = os.path.join(self._export_dir, "html", suite_name, component)
export_dir_issues = os.path.join(export_dir_section, "issues")
export_dir_metainfo = os.path.join(export_dir_section, "metainfo")
error_count = 0
warning_count = 0
info_count = 0
metainfo_count = 0
hint_pages = dict()
cpt_pages = dict()
for arch in suite['architectures']:
h_fname = os.path.join(dep11_hintsdir, suite_name, component, "DEP11Hints_%s.yml.gz" % (arch))
hints_data = None
if os.path.isfile(h_fname):
f = gzip.open(h_fname, 'r')
hints_data = yaml.safe_load_all(f.read())
f.close()
d_fname = os.path.join(dep11_minfodir, suite_name, component, "Components-%s.yml.gz" % (arch))
dep11_data = None
if os.path.isfile(d_fname):
f = gzip.open(d_fname, 'r')
dep11_data = yaml.safe_load_all(f.read())
f.close()
pkg_index = read_packages_dict_from_file(self._archive_root, suite_name, component, arch)
if hints_data:
for hdata in hints_data:
pkg_name = hdata['Package']
pkg_id = hdata.get('PackageID')
if not pkg_id:
pkg_id = pkg_name
pkg = pkg_index.get(pkg_name)
maintainer = None
if pkg:
maintainer = pkg['maintainer']
if not maintainer:
maintainer = "Unknown"
if not issue_summaries.get(maintainer):
issue_summaries[maintainer] = dict()
hints_raw = hdata.get('Hints', list())
# expand all hints to show long descriptions
errors = list()
warnings = list()
infos = list()
for hint in hints_raw:
ehint = self._expand_hint(hint)
severity = ehint['severity']
if severity == "info":
infos.append(ehint)
elif severity == "warning":
warnings.append(ehint)
else:
errors.append(ehint)
if not hint_pages.get(pkg_name):
hint_pages[pkg_name] = list()
# we fold multiple architectures with the same issues into one view
pkid_noarch = pkg_id
if "/" in pkg_id:
pkid_noarch = pkg_id[:pkg_id.rfind("/")]
pcid = ""
if hdata.get('ID'):
pcid = "%s: %s" % (pkid_noarch, hdata.get('ID'))
else:
pcid = pkid_noarch
page_data = {'identifier': pcid, 'errors': errors, 'warnings': warnings, 'infos': infos, 'archs': [arch]}
try:
l = hint_pages[pkg_name]
index = next(i for i, v in enumerate(l) if equal_dicts(v, page_data, ['archs']))
hint_pages[pkg_name][index]['archs'].append(arch)
except StopIteration:
hint_pages[pkg_name].append(page_data)
# add info to global issue count
error_count += len(errors)
warning_count += len(warnings)
info_count += len(infos)
# add info for global index
if not issue_summaries[maintainer].get(pkg_name):
issue_summaries[maintainer][pkg_name] = {'error_count': len(errors), 'warning_count': len(warnings), 'info_count': len(infos)}
if dep11_data:
for mdata in dep11_data:
pkg_name = mdata.get('Package')
if not pkg_name:
# we probably hit the header
continue
pkg = pkg_index.get(pkg_name)
maintainer = None
if pkg:
maintainer = pkg['maintainer']
if not maintainer:
maintainer = "Unknown"
if not mdata_summaries.get(maintainer):
mdata_summaries[maintainer] = dict()
# ugly hack to have the screenshot entries linked
#if mdata.get('Screenshots'):
# sshot_baseurl = os.path.join(self._dep11_url, component)
# for i in range(len(mdata['Screenshots'])):
# url = mdata['Screenshots'][i]['source-image']['url']
# url = "<a href=\"%s\">%s</a>" % (os.path.join(sshot_baseurl, url), url)
# mdata['Screenshots'][i]['source-image']['url'] = Markup(url)
# thumbnails = mdata['Screenshots'][i]['thumbnails']
# for j in range(len(thumbnails)):
# url = thumbnails[j]['url']
# url = "<a href=\"%s\">%s</a>" % (os.path.join(sshot_baseurl, url), url)
# thumbnails[j]['url'] = Markup(url)
# mdata['Screenshots'][i]['thumbnails'] = thumbnails
mdata_yml = dict_to_dep11_yaml(mdata)
mdata_yml = self._highlight_yaml(mdata_yml)
cid = mdata.get('ID')
# try to find an icon for this component (if it's a GUI app)
icon_url = None
if mdata['Type'] == 'desktop-app' or mdata['Type'] == "web-app":
icon_name = mdata['Icon'].get("cached")
cptgid = build_cpt_global_id(cid, mdata.get('X-Source-Checksum'))
if icon_name and cptgid:
icon_fname = os.path.join(component, cptgid, "icons", "64x64", icon_name)
if os.path.isfile(os.path.join(media_dir, icon_fname)):
icon_url = os.path.join(self._dep11_url, icon_fname)
else:
icon_url = noimage_url
else:
icon_url = noimage_url
else:
icon_url = os.path.join(self._html_url, "static", "img", "cpt-nogui.png")
if not cpt_pages.get(pkg_name):
cpt_pages[pkg_name] = list()
page_data = {'cid': cid, 'mdata': mdata_yml, 'icon_url': icon_url, 'archs': [arch]}
try:
l = cpt_pages[pkg_name]
index = next(i for i, v in enumerate(l) if equal_dicts(v, page_data, ['archs']))
cpt_pages[pkg_name][index]['archs'].append(arch)
except StopIteration:
cpt_pages[pkg_name].append(page_data)
# increase valid metainfo count
metainfo_count += 1
# check if we had this package, and add to summary
pksum = mdata_summaries[maintainer].get(pkg_name)
if not pksum:
pksum = dict()
if pksum.get('cids'):
if not cid in pksum['cids']:
pksum['cids'].append(cid)
else:
pksum['cids'] = [cid]
mdata_summaries[maintainer][pkg_name] = pksum
if not dep11_data and not hints_data:
log.warning("Suite %s/%s/%s does not contain DEP-11 data or issue hints.", suite_name, component, arch)
# now write the HTML pages with the previously collected & transformed issue data
for pkg_name, entry_list in hint_pages.items():
# render issues page
self.render_template("issues_page.html", export_dir_issues, "%s.html" % (pkg_name),
package_name=pkg_name, entries=entry_list, suite=suite_name, section=component)
# render page with all components found in a package
for pkg_name, cptlist in cpt_pages.items():
# render metainfo page
self.render_template("metainfo_page.html", export_dir_metainfo, "%s.html" % (pkg_name),
package_name=pkg_name, cpts=cptlist, suite=suite_name, section=component)
# Now render our issue index page
self.render_template("issues_index.html", export_dir_issues, "index.html",
package_summaries=issue_summaries, suite=suite_name, section=component)
# ... and the metainfo index page
self.render_template("metainfo_index.html", export_dir_metainfo, "index.html",
package_summaries=mdata_summaries, suite=suite_name, section=component)
validate_result = "Validation was not performed."
if dep11_data:
# do format validation
validator = DEP11Validator()
ret = validator.validate_file(d_fname)
if ret:
validate_result = "No errors found."
else:
validate_result = ""
for issue in validator.issue_list:
validate_result += issue.replace("FATAL", "<strong>FATAL</strong>")+"<br/>\n"
# sum up counts for suite statistics
suite_metainfo_count += metainfo_count
suite_error_count += error_count
suite_warning_count += warning_count
suite_info_count += info_count
# calculate statistics for this component
count = metainfo_count + error_count + warning_count + info_count
valid_perc = 100/count*metainfo_count if count > 0 else 0
error_perc = 100/count*error_count if count > 0 else 0
warning_perc = 100/count*warning_count if count > 0 else 0
info_perc = 100/count*info_count if count > 0 else 0
# Render our overview page
self.render_template("section_overview.html", export_dir_section, "index.html",
suite=suite_name, section=component, valid_percentage=valid_perc,
error_percentage=error_perc, warning_percentage=warning_perc, info_percentage=info_perc,
metainfo_count=metainfo_count, error_count=error_count, warning_count=warning_count,
info_count=info_count, validate_result=validate_result)
# calculate statistics for this suite
count = suite_metainfo_count + suite_error_count + suite_warning_count + suite_info_count
valid_perc = 100/count*suite_metainfo_count if count > 0 else 0
error_perc = 100/count*suite_error_count if count > 0 else 0
warning_perc = 100/count*suite_warning_count if count > 0 else 0
info_perc = 100/count*suite_info_count if count > 0 else 0
# Render archive components index/overview page
self.render_template("sections_index.html", export_dir, "index.html",
sections=suite['components'], suite=suite_name, valid_percentage=valid_perc,
error_percentage=error_perc, warning_percentage=warning_perc, info_percentage=info_perc,
metainfo_count=suite_metainfo_count, error_count=suite_error_count, warning_count=suite_warning_count,
info_count=suite_info_count)
# Copy the static files
target_static_dir = os.path.join(self._export_dir, "html", "static")
shutil.rmtree(target_static_dir, ignore_errors=True)
shutil.copytree(os.path.join(self._template_dir, "static"), target_static_dir)
def _get_packages_for(self, suite, component, arch):
return read_packages_dict_from_file(self._archive_root, suite, component, arch).values()