def add(self, fetcher):
self.total += 1
url_type = fetcher.url_attr
self.url_type[url_type or 'no code'] += 1
if url_type == 'url':
digest = getattr(fetcher, 'digest', None)
if digest:
algo = crypto.hash_algo_for_digest(digest)
else:
algo = 'no checksum'
self.checksums[algo] += 1
# parse out the URL scheme (https/http/ftp/etc.)
urlinfo = urllib_parse.urlparse(fetcher.url)
self.schemes[urlinfo.scheme] += 1
elif url_type == 'git':
if getattr(fetcher, 'commit', None):
self.git_type['commit'] += 1
elif getattr(fetcher, 'branch', None):
self.git_type['branch'] += 1
elif getattr(fetcher, 'tag', None):
self.git_type['tag'] += 1
else:
self.git_type['no ref'] += 1
def add(self, pkg_name, fetcher):
self.total += 1
url_type = fetcher.url_attr
self.url_type[url_type or 'no code'] += 1
if url_type == 'url':
digest = getattr(fetcher, 'digest', None)
if digest:
algo = crypto.hash_algo_for_digest(digest)
else:
algo = 'no checksum'
self.checksums[algo] += 1
if algo == "md5":
md5_hashes = issues["md5 hashes"]
md5_hashes[pkg_name].append(fetcher.url)
# parse out the URL scheme (https/http/ftp/etc.)
urlinfo = urllib_parse.urlparse(fetcher.url)
self.schemes[urlinfo.scheme] += 1
if urlinfo.scheme == "http":
http_urls = issues["http urls"]
http_urls[pkg_name].append(fetcher.url)
elif url_type == 'git':
if getattr(fetcher, 'commit', None):
self.git_type['commit'] += 1
elif getattr(fetcher, 'branch', None):
self.git_type['branch'] += 1
elif getattr(fetcher, 'tag', None):
self.git_type['tag'] += 1
else:
self.git_type['no ref'] += 1
def invalid_sha256_digest(fetcher):
if getattr(fetcher, "digest", None):
h = crypto.hash_algo_for_digest(fetcher.digest)
if h != "sha256":
return h
def url_stats(args):
stats = {} # stats about fetchers in packages.
nvers = 0 # total number of versions
npkgs = 0 # total number of packages
def inc(fstype, category, attr=None):
"""Increment statistics in the stats dict."""
categories = stats.setdefault(fstype, {})
if attr:
cat_stats = categories.setdefault(category, {})
val = cat_stats.setdefault(attr, 0)
stats[fstype][category][attr] = val + 1
else:
val = categories.setdefault(category, 0)
stats[fstype][category] = val + 1
# over all packages
for pkg in spack.repo.path.all_packages():
npkgs += 1
# look at each version
for v, args in pkg.versions.items():
# figure out what type of fetcher it is
fetcher = fs.for_package_version(pkg, v)
nvers += 1
fstype = fetcher.url_attr
inc(fstype, 'total')
# put some special stats in for particular types of fetchers.
if fstype == 'git':
if 'commit' in args:
inc('git', 'security', 'commit')
else:
inc('git', 'security', 'no commit')
elif fstype == 'url':
for h in crypto.hashes:
if h in args:
inc('url', 'checksums', h)
break
else:
if 'checksum' in args:
h = crypto.hash_algo_for_digest(args['checksum'])
inc('url', 'checksums', h)
else:
inc('url', 'checksums', 'no checksum')
# parse out the URL scheme (https/http/ftp/etc.)
urlinfo = urlparse(fetcher.url)
inc('url', 'schemes', urlinfo.scheme)
# print a nice summary table
tty.msg("%d total versions for %d packages:" % (nvers, npkgs))
line_width = 36
print("-" * line_width)
for fetcher, fetcher_stats in sorted(stats.items(), reverse=True):
fs_total = fetcher_stats['total']
fs_pct = float(fs_total) / nvers * 100
print("%-22s%5d%8.1f%%" % (fetcher, fs_total, fs_pct))
for category, cat_stats in sorted(fetcher_stats.items(), reverse=True):
if category == 'total':
continue
print(" %s" % category)
for name, number in sorted(cat_stats.items(), reverse=True):
pct = float(number) / fs_total * 100
print(" %-18s%5d%8.1f%%" % (name, number, pct))
print("-" * line_width)
