def test_url_parse_offset(name, noffset, ver, voffset, path): """Tests that the name, version and offsets are computed correctly. Args: name (str): expected name noffset (int): name offset ver (str): expected version voffset (int): version offset path (str): url to be parsed """ # Make sure parse_name_offset and parse_name_version are working v, vstart, vlen, vi, vre = parse_version_offset(path) n, nstart, nlen, ni, nre = parse_name_offset(path, v) assert n == name assert v == ver assert nstart == noffset assert vstart == voffset
def url_parse(args): url = args.url tty.msg('Parsing URL: {0}'.format(url)) print() ver, vs, vl, vi, vregex = parse_version_offset(url) tty.msg('Matched version regex {0:>2}: r{1!r}'.format(vi, vregex)) name, ns, nl, ni, nregex = parse_name_offset(url, ver) tty.msg('Matched name regex {0:>2}: r{1!r}'.format(ni, nregex)) print() tty.msg('Detected:') try: print_name_and_version(url) except UrlParseError as e: tty.error(str(e)) print(' name: {0}'.format(name)) print(' version: {0}'.format(ver)) print() tty.msg('Substituting version 9.9.9b:') newurl = substitute_version(url, '9.9.9b') print_name_and_version(newurl) if args.spider: print() tty.msg('Spidering for versions:') versions = find_versions_of_archive(url) if not versions: print(' Found no versions for {0}'.format(name)) return max_len = max(len(str(v)) for v in versions) for v in sorted(versions): print('{0:{1}} {2}'.format(v, max_len, versions[v]))
def url_summary(args): # Collect statistics on how many URLs were correctly parsed total_urls = 0 correct_names = 0 correct_versions = 0 # Collect statistics on which regexes were matched and how often name_regex_dict = dict() right_name_count = defaultdict(int) wrong_name_count = defaultdict(int) version_regex_dict = dict() right_version_count = defaultdict(int) wrong_version_count = defaultdict(int) tty.msg('Generating a summary of URL parsing in Spack...') # Loop through all packages for pkg in spack.repo.path.all_packages(): urls = set() url = getattr(pkg, 'url', None) if url: urls.add(url) for params in pkg.versions.values(): url = params.get('url', None) if url: urls.add(url) # Calculate statistics for url in urls: total_urls += 1 # Parse versions version = None try: version, vs, vl, vi, vregex = parse_version_offset(url) version_regex_dict[vi] = vregex if version_parsed_correctly(pkg, version): correct_versions += 1 right_version_count[vi] += 1 else: wrong_version_count[vi] += 1 except UndetectableVersionError: pass # Parse names try: name, ns, nl, ni, nregex = parse_name_offset(url, version) name_regex_dict[ni] = nregex if name_parsed_correctly(pkg, name): correct_names += 1 right_name_count[ni] += 1 else: wrong_name_count[ni] += 1 except UndetectableNameError: pass print() print(' Total URLs found: {0}'.format(total_urls)) print(' Names correctly parsed: {0:>4}/{1:>4} ({2:>6.2%})'.format( correct_names, total_urls, correct_names / total_urls)) print(' Versions correctly parsed: {0:>4}/{1:>4} ({2:>6.2%})'.format( correct_versions, total_urls, correct_versions / total_urls)) print() tty.msg('Statistics on name regular expressions:') print() print(' Index Right Wrong Total Regular Expression') for ni in sorted(name_regex_dict.keys()): print(' {0:>5} {1:>5} {2:>5} {3:>5} r{4!r}'.format( ni, right_name_count[ni], wrong_name_count[ni], right_name_count[ni] + wrong_name_count[ni], name_regex_dict[ni])) print() tty.msg('Statistics on version regular expressions:') print() print(' Index Right Wrong Total Regular Expression') for vi in sorted(version_regex_dict.keys()): print(' {0:>5} {1:>5} {2:>5} {3:>5} r{4!r}'.format( vi, right_version_count[vi], wrong_version_count[vi], right_version_count[vi] + wrong_version_count[vi], version_regex_dict[vi])) print() # Return statistics, only for testing purposes return (total_urls, correct_names, correct_versions, right_name_count, right_version_count)