Beispiel #1
0
def fetch_tarballs(url, name, version):
    """Try to find versions of the supplied archive by scraping the web.
    Prompts the user to select how many to download if many are found."""
    versions = spack.util.web.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = {version: url}
    elif len(versions) > 1:
        tty.msg("Found %s versions of %s:" % (len(versions), name),
                *spack.cmd.elide_list(
                    ["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
        print('')
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5, abort='q')

        if not archives_to_fetch:
            tty.die("Aborted.")

    sorted_versions = sorted(versions.keys(), reverse=True)
    sorted_urls = [versions[v] for v in sorted_versions]
    return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
Beispiel #2
0
def fetch_tarballs(url, name, version):
    """Try to find versions of the supplied archive by scraping the web.

    Prompts the user to select how many to download if many are found.


    """
    versions = spack.util.web.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = { version : url }
    elif len(versions) > 1:
        tty.msg("Found %s versions of %s:" % (len(versions), name),
                *spack.cmd.elide_list(
                    ["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
        print
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5, abort='q')

        if not archives_to_fetch:
            tty.die("Aborted.")

    sorted_versions = sorted(versions.keys(), reverse=True)
    sorted_urls = [versions[v] for v in sorted_versions]
    return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
Beispiel #3
0
def checksum(parser, args):
    # get the package we're going to generate checksums for
    pkg = spack.repo.get(args.package)

    # If the user asked for specific versions, use those.
    if args.versions:
        versions = {}
        for v in args.versions:
            v = ver(v)
            if not isinstance(v, Version):
                tty.die("Cannot generate checksums for version lists or " +
                        "version ranges.  Use unambiguous versions.")
            versions[v] = pkg.url_for_version(v)
    else:
        versions = pkg.fetch_remote_versions()
        if not versions:
            tty.die("Could not fetch any versions for %s" % pkg.name)

    sorted_versions = sorted(versions, reverse=True)

    tty.msg(
        "Found %s versions of %s" % (len(versions), pkg.name),
        *spack.cmd.elide_list(
            ["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
    print
    archives_to_fetch = tty.get_number("How many would you like to checksum?",
                                       default=5,
                                       abort='q')

    if not archives_to_fetch:
        tty.msg("Aborted.")
        return

    version_hashes = get_checksums(
        sorted_versions[:archives_to_fetch],
        [versions[v] for v in sorted_versions[:archives_to_fetch]],
        keep_stage=args.keep_stage)

    if not version_hashes:
        tty.die("Could not fetch any versions for %s" % pkg.name)

    version_lines = [
        "    version('%s', '%s')" % (v, h) for v, h in version_hashes
    ]
    tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)
Beispiel #4
0
def checksum(parser, args):
    # get the package we're going to generate checksums for
    pkg = spack.db.get(args.package)

    # If the user asked for specific versions, use those.
    versions = [ver(v) for v in args.versions]

    if not all(type(v) == Version for v in versions):
        tty.die("Cannot generate checksums for version lists or " +
                "version ranges.  Use unambiguous versions.")

    if not versions:
        versions = pkg.fetch_available_versions()
        if not versions:
            tty.die("Could not fetch any available versions for %s." %
                    pkg.name)

    versions = list(reversed(versions))
    urls = [pkg.url_for_version(v) for v in versions]

    tty.msg(
        "Found %s versions of %s." % (len(urls), pkg.name),
        *spack.cmd.elide_list(
            ["%-10s%s" % (v, u) for v, u in zip(versions, urls)]))
    print
    archives_to_fetch = tty.get_number("How many would you like to checksum?",
                                       default=5,
                                       abort='q')

    if not archives_to_fetch:
        tty.msg("Aborted.")
        return

    version_hashes = get_checksums(versions[:archives_to_fetch],
                                   urls[:archives_to_fetch],
                                   keep_stage=args.keep_stage)

    if not version_hashes:
        tty.die("Could not fetch any available versions for %s." % pkg.name)

    dict_string = ["    '%s' : '%s'," % (v, h) for v, h in version_hashes]
    dict_string = ['{'] + dict_string + ["}"]

    tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string)
Beispiel #5
0
def checksum(parser, args):
    # get the package we're going to generate checksums for
    pkg = spack.db.get(args.package)

    # If the user asked for specific versions, use those.
    versions = [ver(v) for v in args.versions]

    if not all(type(v) == Version for v in versions):
        tty.die("Cannot generate checksums for version lists or " +
                "version ranges.  Use unambiguous versions.")

    if not versions:
        versions = pkg.fetch_available_versions()
        if not versions:
            tty.die("Could not fetch any available versions for %s." % pkg.name)

    versions = list(reversed(versions))
    urls = [pkg.url_for_version(v) for v in versions]


    tty.msg("Found %s versions of %s." % (len(urls), pkg.name),
            *spack.cmd.elide_list(
            ["%-10s%s" % (v,u) for v, u in zip(versions, urls)]))
    print
    archives_to_fetch = tty.get_number(
        "How many would you like to checksum?", default=5, abort='q')

    if not archives_to_fetch:
        tty.msg("Aborted.")
        return

    version_hashes = get_checksums(
        versions[:archives_to_fetch], urls[:archives_to_fetch], keep_stage=args.keep_stage)

    if not version_hashes:
        tty.die("Could not fetch any available versions for %s." % pkg.name)

    dict_string = ["    '%s' : '%s'," % (v, h) for v, h in version_hashes]
    dict_string = ['{'] + dict_string + ["}"]

    tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string)
Beispiel #6
0
def checksum(parser, args):
    # get the package we're going to generate checksums for
    pkg = spack.repo.get(args.package)

    # If the user asked for specific versions, use those.
    if args.versions:
        versions = {}
        for v in args.versions:
            v = ver(v)
            if not isinstance(v, Version):
                tty.die("Cannot generate checksums for version lists or " +
                        "version ranges.  Use unambiguous versions.")
            versions[v] = pkg.url_for_version(v)
    else:
        versions = pkg.fetch_remote_versions()
        if not versions:
            tty.die("Could not fetch any versions for %s" % pkg.name)

    sorted_versions = sorted(versions, reverse=True)

    tty.msg("Found %s versions of %s" % (len(versions), pkg.name),
            *spack.cmd.elide_list(
                ["%-10s%s" % (v, versions[v]) for v in sorted_versions]))
    print
    archives_to_fetch = tty.get_number(
        "How many would you like to checksum?", default=5, abort='q')

    if not archives_to_fetch:
        tty.msg("Aborted.")
        return

    version_hashes = get_checksums(
        sorted_versions[:archives_to_fetch],
        [versions[v] for v in sorted_versions[:archives_to_fetch]],
        keep_stage=args.keep_stage)

    if not version_hashes:
        tty.die("Could not fetch any versions for %s" % pkg.name)

    version_lines = ["    version('%s', '%s')" % (v, h) for v, h in version_hashes]
    tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)
Beispiel #7
0
def get_checksums(url_dict, name, **kwargs):
    """Fetches and checksums archives from URLs.

    This function is called by both ``spack checksum`` and ``spack create``.
    The ``first_stage_function`` kwarg allows ``spack create`` to determine
    things like the build system of the archive.

    Args:
        url_dict (dict): A dictionary of the form: version -> URL
        name (str): The name of the package
        first_stage_function (callable): Function to run on first staging area
        keep_stage (bool): Don't clean up staging area when command completes

    Returns:
        str: A multi-line string containing versions and corresponding hashes
    """
    first_stage_function = kwargs.get('first_stage_function', None)
    keep_stage = kwargs.get('keep_stage', False)

    sorted_versions = sorted(url_dict.keys(), reverse=True)

    # Find length of longest string in the list for padding
    max_len = max(len(str(v)) for v in sorted_versions)
    num_ver = len(sorted_versions)

    tty.msg("Found {0} version{1} of {2}:".format(
            num_ver, '' if num_ver == 1 else 's', name),
            "",
            *spack.cmd.elide_list(
                ["{0:{1}}  {2}".format(str(v), max_len, url_dict[v])
                 for v in sorted_versions]))
    print()

    archives_to_fetch = tty.get_number(
        "How many would you like to checksum?", default=1, abort='q')

    if not archives_to_fetch:
        tty.die("Aborted.")

    versions = sorted_versions[:archives_to_fetch]
    urls = [url_dict[v] for v in versions]

    tty.msg("Downloading...")
    version_hashes = []
    i = 0
    for url, version in zip(urls, versions):
        try:
            with Stage(url, keep=keep_stage) as stage:
                # Fetch the archive
                stage.fetch()
                if i == 0 and first_stage_function:
                    # Only run first_stage_function the first time,
                    # no need to run it every time
                    first_stage_function(stage, url)

                # Checksum the archive and add it to the list
                version_hashes.append((version, spack.util.crypto.checksum(
                    hashlib.md5, stage.archive_file)))
                i += 1
        except FailedDownloadError:
            tty.msg("Failed to fetch {0}".format(url))
        except Exception as e:
            tty.msg("Something failed on {0}, skipping.".format(url),
                    "  ({0})".format(e))

    if not version_hashes:
        tty.die("Could not fetch any versions for {0}".format(name))

    # Find length of longest string in the list for padding
    max_len = max(len(str(v)) for v, h in version_hashes)

    # Generate the version directives to put in a package.py
    version_lines = "\n".join([
        "    version('{0}', {1}'{2}')".format(
            v, ' ' * (max_len - len(str(v))), h) for v, h in version_hashes
    ])

    num_hash = len(version_hashes)
    tty.msg("Checksummed {0} version{1} of {2}".format(
        num_hash, '' if num_hash == 1 else 's', name))

    return version_lines
Beispiel #8
0
def get_checksums_for_versions(url_dict, name, **kwargs):
    """Fetches and checksums archives from URLs.

    This function is called by both ``spack checksum`` and ``spack
    create``.  The ``first_stage_function`` argument allows the caller to
    inspect the first downloaded archive, e.g., to determine the build
    system.

    Args:
        url_dict (dict): A dictionary of the form: version -> URL
        name (str): The name of the package
        first_stage_function (typing.Callable): function that takes a Stage and a URL;
            this is run on the stage of the first URL downloaded
        keep_stage (bool): whether to keep staging area when command completes
        batch (bool): whether to ask user how many versions to fetch (false)
            or fetch all versions (true)
        latest (bool): whether to take the latest version (true) or all (false)
        fetch_options (dict): Options used for the fetcher (such as timeout
            or cookies)

    Returns:
        (str): A multi-line string containing versions and corresponding hashes

    """
    batch = kwargs.get('batch', False)
    fetch_options = kwargs.get('fetch_options', None)
    first_stage_function = kwargs.get('first_stage_function', None)
    keep_stage = kwargs.get('keep_stage', False)
    latest = kwargs.get('latest', False)

    sorted_versions = sorted(url_dict.keys(), reverse=True)
    if latest:
        sorted_versions = sorted_versions[:1]

    # Find length of longest string in the list for padding
    max_len = max(len(str(v)) for v in sorted_versions)
    num_ver = len(sorted_versions)

    tty.msg(
        'Found {0} version{1} of {2}:'.format(num_ver,
                                              '' if num_ver == 1 else 's',
                                              name), '',
        *llnl.util.lang.elide_list([
            '{0:{1}}  {2}'.format(str(v), max_len, url_dict[v])
            for v in sorted_versions
        ]))
    print()

    if batch or latest:
        archives_to_fetch = len(sorted_versions)
    else:
        archives_to_fetch = tty.get_number(
            "How many would you like to checksum?", default=1, abort='q')

    if not archives_to_fetch:
        tty.die("Aborted.")

    versions = sorted_versions[:archives_to_fetch]
    urls = [url_dict[v] for v in versions]

    tty.debug('Downloading...')
    version_hashes = []
    i = 0
    errors = []
    for url, version in zip(urls, versions):
        # Wheels should not be expanded during staging
        expand_arg = ''
        if url.endswith('.whl') or '.whl#' in url:
            expand_arg = ', expand=False'
        try:
            if fetch_options:
                url_or_fs = fs.URLFetchStrategy(url,
                                                fetch_options=fetch_options)
            else:
                url_or_fs = url
            with Stage(url_or_fs, keep=keep_stage) as stage:
                # Fetch the archive
                stage.fetch()
                if i == 0 and first_stage_function:
                    # Only run first_stage_function the first time,
                    # no need to run it every time
                    first_stage_function(stage, url)

                # Checksum the archive and add it to the list
                version_hashes.append(
                    (version,
                     spack.util.crypto.checksum(hashlib.sha256,
                                                stage.archive_file)))
                i += 1
        except FailedDownloadError:
            errors.append('Failed to fetch {0}'.format(url))
        except Exception as e:
            tty.msg('Something failed on {0}, skipping.  ({1})'.format(url, e))

    for msg in errors:
        tty.debug(msg)

    if not version_hashes:
        tty.die("Could not fetch any versions for {0}".format(name))

    # Find length of longest string in the list for padding
    max_len = max(len(str(v)) for v, h in version_hashes)

    # Generate the version directives to put in a package.py
    version_lines = "\n".join([
        "    version('{0}', {1}sha256='{2}'{3})".format(
            v, ' ' * (max_len - len(str(v))), h, expand_arg)
        for v, h in version_hashes
    ])

    num_hash = len(version_hashes)
    tty.debug('Checksummed {0} version{1} of {2}:'.format(
        num_hash, '' if num_hash == 1 else 's', name))

    return version_lines
Beispiel #9
0
    versions = spack.util.web.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = { version : url }
    elif len(versions) > 1:
        tty.msg("Found %s versions of %s:" % (len(versions), name),
                *spack.cmd.elide_list(
                    ["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
        print
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5, abort='q')

        if not archives_to_fetch:
            tty.msg("Aborted.")
            return

    guesser = ConfigureGuesser()
    ver_hash_tuples = spack.cmd.checksum.get_checksums(
        versions.keys()[:archives_to_fetch],
        [versions[v] for v in versions.keys()[:archives_to_fetch]],
        first_stage_function=guesser,
        keep_stage=args.keep_stage)

    if not ver_hash_tuples:
        tty.die("Could not fetch any tarballs for %s." % name)
Beispiel #10
0
    versions = spack.package.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = {version: url}
    elif len(versions) > 1:
        tty.msg(
            "Found %s versions of %s:" % (len(versions), name),
            *spack.cmd.elide_list(
                ["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
        print
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5,
            abort='q')

        if not archives_to_fetch:
            tty.msg("Aborted.")
            return

    guesser = ConfigureGuesser()
    ver_hash_tuples = spack.cmd.checksum.get_checksums(
        versions.keys()[:archives_to_fetch],
        [versions[v] for v in versions.keys()[:archives_to_fetch]],
        first_stage_function=guesser,
        keep_stage=args.keep_stage)

    if not ver_hash_tuples:
        tty.die("Could not fetch any tarballs for %s." % name)
Beispiel #11
0
def create(parser, args):
    url = args.url

    # Try to deduce name and version of the new package from the URL
    name, version = spack.url.parse_name_and_version(url)
    if not name:
        tty.msg("Couldn't guess a name for this package.")
        name = get_name()

    if not version:
        tty.die("Couldn't guess a version string from %s." % url)

    tty.msg("This looks like a URL for %s version %s." % (name, version))
    tty.msg("Creating template for package %s" % name)

    # Create a directory for the new package.
    pkg_path = spack.db.filename_for_package_name(name)
    if os.path.exists(pkg_path) and not args.force:
        tty.die("%s already exists." % pkg_path)
    else:
        mkdirp(os.path.dirname(pkg_path))

    versions = list(reversed(spack.package.find_versions_of_archive(url)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = [version]
        urls = [url]
    else:
        urls = [spack.url.substitute_version(url, v) for v in versions]
        if len(urls) > 1:
            tty.msg(
                "Found %s versions of %s:" % (len(urls), name),
                *spack.cmd.elide_list(["%-10s%s" % (v, u) for v, u in zip(versions, urls)])
            )
            print
            archives_to_fetch = tty.get_number("Include how many checksums in the package file?", default=5, abort="q")

            if not archives_to_fetch:
                tty.msg("Aborted.")
                return

    guesser = ConfigureGuesser()
    ver_hash_tuples = spack.cmd.checksum.get_checksums(
        versions[:archives_to_fetch], urls[:archives_to_fetch], first_stage_function=guesser, keep_stage=args.keep_stage
    )

    if not ver_hash_tuples:
        tty.die("Could not fetch any tarballs for %s." % name)

    # Write out a template for the file
    with closing(open(pkg_path, "w")) as pkg_file:
        pkg_file.write(
            package_template.substitute(
                name=name,
                configure=guesser.configure,
                class_name=mod_to_class(name),
                url=url,
                versions=make_version_dict(ver_hash_tuples),
            )
        )

    # If everything checks out, go ahead and edit.
    spack.editor(pkg_path)
    tty.msg("Created package %s." % pkg_path)