def fetch_tarballs(url, name, version):
"""Try to find versions of the supplied archive by scraping the web.
Prompts the user to select how many to download if many are found."""
versions = spack.util.web.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = {version: url}
elif len(versions) > 1:
tty.msg("Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
print('')
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
tty.die("Aborted.")
sorted_versions = sorted(versions.keys(), reverse=True)
sorted_urls = [versions[v] for v in sorted_versions]
return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
def fetch_tarballs(url, name, version):
"""Try to find versions of the supplied archive by scraping the web.
Prompts the user to select how many to download if many are found.
"""
versions = spack.util.web.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = { version : url }
elif len(versions) > 1:
tty.msg("Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
tty.die("Aborted.")
sorted_versions = sorted(versions.keys(), reverse=True)
sorted_urls = [versions[v] for v in sorted_versions]
return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
def checksum(parser, args):
# get the package we're going to generate checksums for
pkg = spack.repo.get(args.package)
# If the user asked for specific versions, use those.
if args.versions:
versions = {}
for v in args.versions:
v = ver(v)
if not isinstance(v, Version):
tty.die("Cannot generate checksums for version lists or " +
"version ranges. Use unambiguous versions.")
versions[v] = pkg.url_for_version(v)
else:
versions = pkg.fetch_remote_versions()
if not versions:
tty.die("Could not fetch any versions for %s" % pkg.name)
sorted_versions = sorted(versions, reverse=True)
tty.msg(
"Found %s versions of %s" % (len(versions), pkg.name),
*spack.cmd.elide_list(
["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
print
archives_to_fetch = tty.get_number("How many would you like to checksum?",
default=5,
abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
version_hashes = get_checksums(
sorted_versions[:archives_to_fetch],
[versions[v] for v in sorted_versions[:archives_to_fetch]],
keep_stage=args.keep_stage)
if not version_hashes:
tty.die("Could not fetch any versions for %s" % pkg.name)
version_lines = [
" version('%s', '%s')" % (v, h) for v, h in version_hashes
]
tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)
def checksum(parser, args):
# get the package we're going to generate checksums for
pkg = spack.db.get(args.package)
# If the user asked for specific versions, use those.
versions = [ver(v) for v in args.versions]
if not all(type(v) == Version for v in versions):
tty.die("Cannot generate checksums for version lists or " +
"version ranges. Use unambiguous versions.")
if not versions:
versions = pkg.fetch_available_versions()
if not versions:
tty.die("Could not fetch any available versions for %s." %
pkg.name)
versions = list(reversed(versions))
urls = [pkg.url_for_version(v) for v in versions]
tty.msg(
"Found %s versions of %s." % (len(urls), pkg.name),
*spack.cmd.elide_list(
["%-10s%s" % (v, u) for v, u in zip(versions, urls)]))
print
archives_to_fetch = tty.get_number("How many would you like to checksum?",
default=5,
abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
version_hashes = get_checksums(versions[:archives_to_fetch],
urls[:archives_to_fetch],
keep_stage=args.keep_stage)
if not version_hashes:
tty.die("Could not fetch any available versions for %s." % pkg.name)
dict_string = [" '%s' : '%s'," % (v, h) for v, h in version_hashes]
dict_string = ['{'] + dict_string + ["}"]
tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string)
def checksum(parser, args):
# get the package we're going to generate checksums for
pkg = spack.db.get(args.package)
# If the user asked for specific versions, use those.
versions = [ver(v) for v in args.versions]
if not all(type(v) == Version for v in versions):
tty.die("Cannot generate checksums for version lists or " +
"version ranges. Use unambiguous versions.")
if not versions:
versions = pkg.fetch_available_versions()
if not versions:
tty.die("Could not fetch any available versions for %s." % pkg.name)
versions = list(reversed(versions))
urls = [pkg.url_for_version(v) for v in versions]
tty.msg("Found %s versions of %s." % (len(urls), pkg.name),
*spack.cmd.elide_list(
["%-10s%s" % (v,u) for v, u in zip(versions, urls)]))
print
archives_to_fetch = tty.get_number(
"How many would you like to checksum?", default=5, abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
version_hashes = get_checksums(
versions[:archives_to_fetch], urls[:archives_to_fetch], keep_stage=args.keep_stage)
if not version_hashes:
tty.die("Could not fetch any available versions for %s." % pkg.name)
dict_string = [" '%s' : '%s'," % (v, h) for v, h in version_hashes]
dict_string = ['{'] + dict_string + ["}"]
tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string)
def checksum(parser, args):
# get the package we're going to generate checksums for
pkg = spack.repo.get(args.package)
# If the user asked for specific versions, use those.
if args.versions:
versions = {}
for v in args.versions:
v = ver(v)
if not isinstance(v, Version):
tty.die("Cannot generate checksums for version lists or " +
"version ranges. Use unambiguous versions.")
versions[v] = pkg.url_for_version(v)
else:
versions = pkg.fetch_remote_versions()
if not versions:
tty.die("Could not fetch any versions for %s" % pkg.name)
sorted_versions = sorted(versions, reverse=True)
tty.msg("Found %s versions of %s" % (len(versions), pkg.name),
*spack.cmd.elide_list(
["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
print
archives_to_fetch = tty.get_number(
"How many would you like to checksum?", default=5, abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
version_hashes = get_checksums(
sorted_versions[:archives_to_fetch],
[versions[v] for v in sorted_versions[:archives_to_fetch]],
keep_stage=args.keep_stage)
if not version_hashes:
tty.die("Could not fetch any versions for %s" % pkg.name)
version_lines = [" version('%s', '%s')" % (v, h) for v, h in version_hashes]
tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)
def get_checksums(url_dict, name, **kwargs):
"""Fetches and checksums archives from URLs.
This function is called by both ``spack checksum`` and ``spack create``.
The ``first_stage_function`` kwarg allows ``spack create`` to determine
things like the build system of the archive.
Args:
url_dict (dict): A dictionary of the form: version -> URL
name (str): The name of the package
first_stage_function (callable): Function to run on first staging area
keep_stage (bool): Don't clean up staging area when command completes
Returns:
str: A multi-line string containing versions and corresponding hashes
"""
first_stage_function = kwargs.get('first_stage_function', None)
keep_stage = kwargs.get('keep_stage', False)
sorted_versions = sorted(url_dict.keys(), reverse=True)
# Find length of longest string in the list for padding
max_len = max(len(str(v)) for v in sorted_versions)
num_ver = len(sorted_versions)
tty.msg("Found {0} version{1} of {2}:".format(
num_ver, '' if num_ver == 1 else 's', name),
"",
*spack.cmd.elide_list(
["{0:{1}} {2}".format(str(v), max_len, url_dict[v])
for v in sorted_versions]))
print()
archives_to_fetch = tty.get_number(
"How many would you like to checksum?", default=1, abort='q')
if not archives_to_fetch:
tty.die("Aborted.")
versions = sorted_versions[:archives_to_fetch]
urls = [url_dict[v] for v in versions]
tty.msg("Downloading...")
version_hashes = []
i = 0
for url, version in zip(urls, versions):
try:
with Stage(url, keep=keep_stage) as stage:
# Fetch the archive
stage.fetch()
if i == 0 and first_stage_function:
# Only run first_stage_function the first time,
# no need to run it every time
first_stage_function(stage, url)
# Checksum the archive and add it to the list
version_hashes.append((version, spack.util.crypto.checksum(
hashlib.md5, stage.archive_file)))
i += 1
except FailedDownloadError:
tty.msg("Failed to fetch {0}".format(url))
except Exception as e:
tty.msg("Something failed on {0}, skipping.".format(url),
" ({0})".format(e))
if not version_hashes:
tty.die("Could not fetch any versions for {0}".format(name))
# Find length of longest string in the list for padding
max_len = max(len(str(v)) for v, h in version_hashes)
# Generate the version directives to put in a package.py
version_lines = "\n".join([
" version('{0}', {1}'{2}')".format(
v, ' ' * (max_len - len(str(v))), h) for v, h in version_hashes
])
num_hash = len(version_hashes)
tty.msg("Checksummed {0} version{1} of {2}".format(
num_hash, '' if num_hash == 1 else 's', name))
return version_lines
def get_checksums_for_versions(url_dict, name, **kwargs):
"""Fetches and checksums archives from URLs.
This function is called by both ``spack checksum`` and ``spack
create``. The ``first_stage_function`` argument allows the caller to
inspect the first downloaded archive, e.g., to determine the build
system.
Args:
url_dict (dict): A dictionary of the form: version -> URL
name (str): The name of the package
first_stage_function (typing.Callable): function that takes a Stage and a URL;
this is run on the stage of the first URL downloaded
keep_stage (bool): whether to keep staging area when command completes
batch (bool): whether to ask user how many versions to fetch (false)
or fetch all versions (true)
latest (bool): whether to take the latest version (true) or all (false)
fetch_options (dict): Options used for the fetcher (such as timeout
or cookies)
Returns:
(str): A multi-line string containing versions and corresponding hashes
"""
batch = kwargs.get('batch', False)
fetch_options = kwargs.get('fetch_options', None)
first_stage_function = kwargs.get('first_stage_function', None)
keep_stage = kwargs.get('keep_stage', False)
latest = kwargs.get('latest', False)
sorted_versions = sorted(url_dict.keys(), reverse=True)
if latest:
sorted_versions = sorted_versions[:1]
# Find length of longest string in the list for padding
max_len = max(len(str(v)) for v in sorted_versions)
num_ver = len(sorted_versions)
tty.msg(
'Found {0} version{1} of {2}:'.format(num_ver,
'' if num_ver == 1 else 's',
name), '',
*llnl.util.lang.elide_list([
'{0:{1}} {2}'.format(str(v), max_len, url_dict[v])
for v in sorted_versions
]))
print()
if batch or latest:
archives_to_fetch = len(sorted_versions)
else:
archives_to_fetch = tty.get_number(
"How many would you like to checksum?", default=1, abort='q')
if not archives_to_fetch:
tty.die("Aborted.")
versions = sorted_versions[:archives_to_fetch]
urls = [url_dict[v] for v in versions]
tty.debug('Downloading...')
version_hashes = []
i = 0
errors = []
for url, version in zip(urls, versions):
# Wheels should not be expanded during staging
expand_arg = ''
if url.endswith('.whl') or '.whl#' in url:
expand_arg = ', expand=False'
try:
if fetch_options:
url_or_fs = fs.URLFetchStrategy(url,
fetch_options=fetch_options)
else:
url_or_fs = url
with Stage(url_or_fs, keep=keep_stage) as stage:
# Fetch the archive
stage.fetch()
if i == 0 and first_stage_function:
# Only run first_stage_function the first time,
# no need to run it every time
first_stage_function(stage, url)
# Checksum the archive and add it to the list
version_hashes.append(
(version,
spack.util.crypto.checksum(hashlib.sha256,
stage.archive_file)))
i += 1
except FailedDownloadError:
errors.append('Failed to fetch {0}'.format(url))
except Exception as e:
tty.msg('Something failed on {0}, skipping. ({1})'.format(url, e))
for msg in errors:
tty.debug(msg)
if not version_hashes:
tty.die("Could not fetch any versions for {0}".format(name))
# Find length of longest string in the list for padding
max_len = max(len(str(v)) for v, h in version_hashes)
# Generate the version directives to put in a package.py
version_lines = "\n".join([
" version('{0}', {1}sha256='{2}'{3})".format(
v, ' ' * (max_len - len(str(v))), h, expand_arg)
for v, h in version_hashes
])
num_hash = len(version_hashes)
tty.debug('Checksummed {0} version{1} of {2}:'.format(
num_hash, '' if num_hash == 1 else 's', name))
return version_lines
versions = spack.util.web.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = { version : url }
elif len(versions) > 1:
tty.msg("Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5, abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
guesser = ConfigureGuesser()
ver_hash_tuples = spack.cmd.checksum.get_checksums(
versions.keys()[:archives_to_fetch],
[versions[v] for v in versions.keys()[:archives_to_fetch]],
first_stage_function=guesser,
keep_stage=args.keep_stage)
if not ver_hash_tuples:
tty.die("Could not fetch any tarballs for %s." % name)
versions = spack.package.find_versions_of_archive(url)
rkeys = sorted(versions.keys(), reverse=True)
versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = {version: url}
elif len(versions) > 1:
tty.msg(
"Found %s versions of %s:" % (len(versions), name),
*spack.cmd.elide_list(
["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
print
archives_to_fetch = tty.get_number(
"Include how many checksums in the package file?",
default=5,
abort='q')
if not archives_to_fetch:
tty.msg("Aborted.")
return
guesser = ConfigureGuesser()
ver_hash_tuples = spack.cmd.checksum.get_checksums(
versions.keys()[:archives_to_fetch],
[versions[v] for v in versions.keys()[:archives_to_fetch]],
first_stage_function=guesser,
keep_stage=args.keep_stage)
if not ver_hash_tuples:
tty.die("Could not fetch any tarballs for %s." % name)
def create(parser, args):
url = args.url
# Try to deduce name and version of the new package from the URL
name, version = spack.url.parse_name_and_version(url)
if not name:
tty.msg("Couldn't guess a name for this package.")
name = get_name()
if not version:
tty.die("Couldn't guess a version string from %s." % url)
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
# Create a directory for the new package.
pkg_path = spack.db.filename_for_package_name(name)
if os.path.exists(pkg_path) and not args.force:
tty.die("%s already exists." % pkg_path)
else:
mkdirp(os.path.dirname(pkg_path))
versions = list(reversed(spack.package.find_versions_of_archive(url)))
archives_to_fetch = 1
if not versions:
# If the fetch failed for some reason, revert to what the user provided
versions = [version]
urls = [url]
else:
urls = [spack.url.substitute_version(url, v) for v in versions]
if len(urls) > 1:
tty.msg(
"Found %s versions of %s:" % (len(urls), name),
*spack.cmd.elide_list(["%-10s%s" % (v, u) for v, u in zip(versions, urls)])
)
print
archives_to_fetch = tty.get_number("Include how many checksums in the package file?", default=5, abort="q")
if not archives_to_fetch:
tty.msg("Aborted.")
return
guesser = ConfigureGuesser()
ver_hash_tuples = spack.cmd.checksum.get_checksums(
versions[:archives_to_fetch], urls[:archives_to_fetch], first_stage_function=guesser, keep_stage=args.keep_stage
)
if not ver_hash_tuples:
tty.die("Could not fetch any tarballs for %s." % name)
# Write out a template for the file
with closing(open(pkg_path, "w")) as pkg_file:
pkg_file.write(
package_template.substitute(
name=name,
configure=guesser.configure,
class_name=mod_to_class(name),
url=url,
versions=make_version_dict(ver_hash_tuples),
)
)
# If everything checks out, go ahead and edit.
spack.editor(pkg_path)
tty.msg("Created package %s." % pkg_path)