def test_url_parse(): parsed = url_util.parse('/path/to/resource', scheme='fake') assert(parsed.scheme == 'fake') assert(parsed.netloc == '') assert(parsed.path == '/path/to/resource') parsed = url_util.parse('file:///path/to/resource') assert(parsed.scheme == 'file') assert(parsed.netloc == '') assert(parsed.path == '/path/to/resource') parsed = url_util.parse('file:///path/to/resource', scheme='fake') assert(parsed.scheme == 'file') assert(parsed.netloc == '') assert(parsed.path == '/path/to/resource') parsed = url_util.parse('file://path/to/resource') assert(parsed.scheme == 'file') expected = convert_to_posix_path( os.path.abspath( posixpath.join('path', 'to', 'resource'))) if is_windows: expected = expected.lstrip(drive) assert(parsed.path == expected) if is_windows: parsed = url_util.parse('file://%s\\path\\to\\resource' % drive) assert(parsed.scheme == 'file') expected = '/' + posixpath.join('path', 'to', 'resource') assert parsed.path == expected parsed = url_util.parse('https://path/to/resource') assert(parsed.scheme == 'https') assert(parsed.netloc == 'path') assert(parsed.path == '/to/resource') parsed = url_util.parse('gs://path/to/resource') assert(parsed.scheme == 'gs') assert(parsed.netloc == 'path') assert(parsed.path == '/to/resource') spack_root = spack.paths.spack_root parsed = url_util.parse('file://$spack') assert(parsed.scheme == 'file') if is_windows: spack_root = '/' + convert_to_posix_path(spack_root) assert(parsed.netloc + parsed.path == spack_root)
def _std_args(pkg): """Computes the standard cmake arguments for a generic package""" try: generator = pkg.generator except AttributeError: generator = CMakePackage.generator # Make sure a valid generator was chosen valid_primary_generators = ['Unix Makefiles', 'Ninja'] primary_generator = _extract_primary_generator(generator) if primary_generator not in valid_primary_generators: msg = "Invalid CMake generator: '{0}'\n".format(generator) msg += "CMakePackage currently supports the following " msg += "primary generators: '{0}'".\ format("', '".join(valid_primary_generators)) raise InstallError(msg) try: build_type = pkg.spec.variants['build_type'].value except KeyError: build_type = 'RelWithDebInfo' try: ipo = pkg.spec.variants['ipo'].value except KeyError: ipo = False define = CMakePackage.define args = [ '-G', generator, define('CMAKE_INSTALL_PREFIX', convert_to_posix_path(pkg.prefix)), define('CMAKE_BUILD_TYPE', build_type), define('BUILD_TESTING', pkg.run_tests), ] # CMAKE_INTERPROCEDURAL_OPTIMIZATION only exists for CMake >= 3.9 if pkg.spec.satisfies('^[email protected]:'): args.append(define('CMAKE_INTERPROCEDURAL_OPTIMIZATION', ipo)) if primary_generator == 'Unix Makefiles': args.append(define('CMAKE_VERBOSE_MAKEFILE', True)) if platform.mac_ver()[0]: args.extend([ define('CMAKE_FIND_FRAMEWORK', "LAST"), define('CMAKE_FIND_APPBUNDLE', "LAST"), ]) # Set up CMake rpath args.extend([ define('CMAKE_INSTALL_RPATH_USE_LINK_PATH', True), define('CMAKE_INSTALL_RPATH', spack.build_environment.get_rpaths(pkg)), define('CMAKE_PREFIX_PATH', spack.build_environment.get_cmake_prefix_path(pkg)) ]) return args
def setup(self, text): if isinstance(text, string_types): # shlex does not handle Windows path # separators, so we must normalize to posix text = sp.convert_to_posix_path(text) text = shlex.split(str(text)) self.text = text self.push_tokens(self.lexer.lex(text))
def test_root_get_and_set(mutable_config, scope): scope_args, path = [], '/scratch/spack/bootstrap' if scope: scope_args = ['--scope={0}'.format(scope)] _bootstrap('root', path, *scope_args) out = _bootstrap('root', *scope_args, output=str) if sys.platform == 'win32': out = convert_to_posix_path(out) assert out.strip() == path
def _join(base_url, path, *extra, **kwargs): base_url = parse(base_url) resolve_href = kwargs.get('resolve_href', False) (scheme, netloc, base_path, params, query, _) = base_url scheme = scheme.lower() path_tokens = [ part for part in itertools.chain( _split_all(path), itertools.chain.from_iterable( _split_all(extra_path) for extra_path in extra)) if part and part != '/' ] base_path_args = ['/fake-root'] if scheme == 's3': if netloc: base_path_args.append(netloc) if base_path.startswith('/'): base_path = base_path[1:] base_path_args.append(base_path) if resolve_href: new_base_path, _ = posixpath.split(posixpath.join(*base_path_args)) base_path_args = [new_base_path] base_path_args.extend(path_tokens) base_path = posixpath.relpath(posixpath.join(*base_path_args), '/fake-root') if scheme == 's3': path_tokens = [ part for part in _split_all(base_path) if part and part != '/' ] if path_tokens: netloc = path_tokens.pop(0) base_path = posixpath.join('', *path_tokens) if sys.platform == "win32": base_path = convert_to_posix_path(base_path) return format( urllib_parse.ParseResult(scheme=scheme, netloc=netloc, path=base_path, params=params, query=query, fragment=None))
def parse(url, scheme='file'): """Parse a url. For file:// URLs, the netloc and path components are concatenated and passed through spack.util.path.canoncalize_path(). Otherwise, the returned value is the same as urllib's urlparse() with allow_fragments=False. """ # guarantee a value passed in is of proper url format. Guarantee # allows for easier string manipulation accross platforms if isinstance(url, string_types): require_url_format(url) url = escape_file_url(url) url_obj = ( urllib_parse.urlparse(url, scheme=scheme, allow_fragments=False) if isinstance(url, string_types) else url) (scheme, netloc, path, params, query, _) = url_obj scheme = (scheme or 'file').lower() if scheme == 'file': # (The user explicitly provides the file:// scheme.) # examples: # file://C:\\a\\b\\c # file://X:/a/b/c path = canonicalize_path(netloc + path) path = re.sub(r'^/+', '/', path) netloc = '' drive_ltr_lst = re.findall(r'[A-Za-z]:\\', path) is_win_path = bool(drive_ltr_lst) if is_windows and is_win_path: drive_ltr = drive_ltr_lst[0].strip('\\') path = re.sub(r'[\\]*' + drive_ltr, '', path) netloc = '/' + drive_ltr.strip('\\') if sys.platform == "win32": path = convert_to_posix_path(path) return urllib_parse.ParseResult(scheme=scheme, netloc=netloc, path=path, params=params, query=query, fragment=None)
def test_url_join_absolute_paths(): # Handling absolute path components is a little tricky. To this end, we # distinguish "absolute path components", from the more-familiar concept of # "absolute paths" as they are understood for local filesystem paths. # # - All absolute paths are absolute path components. Joining a URL with # these components has the effect of completely replacing the path of the # URL with the absolute path. These components do not specify a URL # scheme, so the scheme of the URL procuced when joining them depend on # those provided by components that came before it (file:// assumed if no # such scheme is provided). # For eaxmple: p = '/path/to/resource' # ...is an absolute path # http:// URL assert (url_util.join('http://example.com/a/b/c', p) == 'http://example.com/path/to/resource') # s3:// URL # also notice how the netloc is treated as part of the path for s3:// URLs assert (url_util.join('s3://example.com/a/b/c', p) == 's3://path/to/resource') # - URL components that specify a scheme are always absolute path # components. Joining a base URL with these components effectively # discards the base URL and "resets" the joining logic starting at the # component in question and using it as the new base URL. # For eaxmple: p = 'http://example.com/path/to' # ...is an http:// URL join_result = url_util.join(p, 'resource') assert (join_result == 'http://example.com/path/to/resource') # works as if everything before the http:// URL was left out assert (url_util.join('literally', 'does', 'not', 'matter', p, 'resource') == join_result) # It's important to keep in mind that this logic applies even if the # component's path is not an absolute path! # For eaxmple: p = './d' # ...is *NOT* an absolute path # ...is also *NOT* an absolute path component u = 'file://./d' # ...is a URL # The path of this URL is *NOT* an absolute path # HOWEVER, the URL, itself, *is* an absolute path component # (We just need... cwd = os.getcwd() # ...to work out what resource it points to) if sys.platform == "win32": convert_to_posix_path(cwd) cwd = '/' + cwd # So, even though parse() assumes "file://" URL, the scheme is still # significant in URL path components passed to join(), even if the base # is a file:// URL. path_join_result = 'file:///a/b/c/d' assert (url_util.join('/a/b/c', p) == path_join_result) assert (url_util.join('file:///a/b/c', p) == path_join_result) url_join_result = 'file://{CWD}/d'.format(CWD=cwd) assert (url_util.join('/a/b/c', u) == url_join_result) assert (url_util.join('file:///a/b/c', u) == url_join_result) # Finally, resolve_href should have no effect for how absolute path # components are handled because local hrefs can not be absolute path # components. args = [ 's3://does', 'not', 'matter', 'http://example.com', 'also', 'does', 'not', 'matter', '/path' ] expected = 'http://example.com/path' assert (url_util.join(*args, resolve_href=True) == expected) assert (url_util.join(*args, resolve_href=False) == expected) # resolve_href only matters for the local path components at the end of the # argument list. args[-1] = '/path/to/page' args.extend(('..', '..', 'resource')) assert (url_util.join(*args, resolve_href=True) == 'http://example.com/resource') assert (url_util.join( *args, resolve_href=False) == 'http://example.com/path/resource')
def join(base_url, path, *extra, **kwargs): """Joins a base URL with one or more local URL path components If resolve_href is True, treat the base URL as though it where the locator of a web page, and the remaining URL path components as though they formed a relative URL to be resolved against it (i.e.: as in posixpath.join(...)). The result is an absolute URL to the resource to which a user's browser would navigate if they clicked on a link with an "href" attribute equal to the relative URL. If resolve_href is False (default), then the URL path components are joined as in posixpath.join(). Note: file:// URL path components are not canonicalized as part of this operation. To canonicalize, pass the joined url to format(). Examples: base_url = 's3://bucket/index.html' body = fetch_body(prefix) link = get_href(body) # link == '../other-bucket/document.txt' # wrong - link is a local URL that needs to be resolved against base_url spack.util.url.join(base_url, link) 's3://bucket/other_bucket/document.txt' # correct - resolve local URL against base_url spack.util.url.join(base_url, link, resolve_href=True) 's3://other_bucket/document.txt' prefix = 'https://mirror.spack.io/build_cache' # wrong - prefix is just a URL prefix spack.util.url.join(prefix, 'my-package', resolve_href=True) 'https://mirror.spack.io/my-package' # correct - simply append additional URL path components spack.util.url.join(prefix, 'my-package', resolve_href=False) # default 'https://mirror.spack.io/build_cache/my-package' # For canonicalizing file:// URLs, take care to explicitly differentiate # between absolute and relative join components. # '$spack' is not an absolute path component join_result = spack.util.url.join('/a/b/c', '$spack') ; join_result 'file:///a/b/c/$spack' spack.util.url.format(join_result) 'file:///a/b/c/opt/spack' # '/$spack' *is* an absolute path component join_result = spack.util.url.join('/a/b/c', '/$spack') ; join_result 'file:///$spack' spack.util.url.format(join_result) 'file:///opt/spack' """ paths = [(x) if isinstance(x, string_types) else x.geturl() for x in itertools.chain((base_url, path), extra)] paths = [convert_to_posix_path(x) for x in paths] n = len(paths) last_abs_component = None scheme = '' for i in range(n - 1, -1, -1): obj = urllib_parse.urlparse(paths[i], scheme='', allow_fragments=False) scheme = obj.scheme # in either case the component is absolute if scheme or obj.path.startswith('/'): if not scheme: # Without a scheme, we have to go back looking for the # next-last component that specifies a scheme. for j in range(i - 1, -1, -1): obj = urllib_parse.urlparse(paths[j], scheme='', allow_fragments=False) if obj.scheme: paths[i] = '{SM}://{NL}{PATH}'.format( SM=obj.scheme, NL=((obj.netloc + '/') if obj.scheme != 's3' else ''), PATH=paths[i][1:]) break last_abs_component = i break if last_abs_component is not None: paths = paths[last_abs_component:] if len(paths) == 1: result = urllib_parse.urlparse(paths[0], scheme='file', allow_fragments=False) # another subtlety: If the last argument to join() is an absolute # file:// URL component with a relative path, the relative path # needs to be resolved. if result.scheme == 'file' and result.netloc: result = urllib_parse.ParseResult( scheme=result.scheme, netloc='', path=posixpath.abspath(result.netloc + result.path), params=result.params, query=result.query, fragment=None) return result.geturl() return _join(*paths, **kwargs)
def read_from_url(url, accept_content_type=None): url = url_util.parse(url) context = None verify_ssl = spack.config.get('config:verify_ssl') # Don't even bother with a context unless the URL scheme is one that uses # SSL certs. if uses_ssl(url): if verify_ssl: if __UNABLE_TO_VERIFY_SSL: # User wants SSL verification, but it cannot be provided. warn_no_ssl_cert_checking() else: # User wants SSL verification, and it *can* be provided. context = ssl.create_default_context() # novm else: # User has explicitly indicated that they do not want SSL # verification. if not __UNABLE_TO_VERIFY_SSL: context = ssl._create_unverified_context() url_scheme = url.scheme url = url_util.format(url) if sys.platform == "win32" and url_scheme == "file": url = convert_to_posix_path(url) req = Request(url) content_type = None is_web_url = url_scheme in ('http', 'https') if accept_content_type and is_web_url: # Make a HEAD request first to check the content type. This lets # us ignore tarballs and gigantic files. # It would be nice to do this with the HTTP Accept header to avoid # one round-trip. However, most servers seem to ignore the header # if you ask for a tarball with Accept: text/html. req.get_method = lambda: "HEAD" resp = _urlopen(req, timeout=_timeout, context=context) content_type = get_header(resp.headers, 'Content-type') # Do the real GET request when we know it's just HTML. req.get_method = lambda: "GET" try: response = _urlopen(req, timeout=_timeout, context=context) except URLError as err: raise SpackWebError('Download failed: {ERROR}'.format(ERROR=str(err))) if accept_content_type and not is_web_url: content_type = get_header(response.headers, 'Content-type') reject_content_type = (accept_content_type and (content_type is None or not content_type.startswith(accept_content_type))) if reject_content_type: tty.debug("ignoring page {0}{1}{2}".format( url, " with content type " if content_type is not None else "", content_type or "")) return None, None, None return response.geturl(), response.headers, response
def find_versions_of_archive(archive_urls, list_url=None, list_depth=0, concurrency=32, reference_package=None): """Scrape web pages for new versions of a tarball. Args: archive_urls (str or list or tuple): URL or sequence of URLs for different versions of a package. Typically these are just the tarballs from the package file itself. By default, this searches the parent directories of archives. list_url (str or None): URL for a listing of archives. Spack will scrape these pages for download links that look like the archive URL. list_depth (int): max depth to follow links on list_url pages. Defaults to 0. concurrency (int): maximum number of concurrent requests reference_package (spack.package.Package or None): a spack package used as a reference for url detection. Uses the url_for_version method on the package to produce reference urls which, if found, are preferred. """ if not isinstance(archive_urls, (list, tuple)): archive_urls = [archive_urls] # Generate a list of list_urls based on archive urls and any # explicitly listed list_url in the package list_urls = set() if list_url is not None: list_urls.add(list_url) for aurl in archive_urls: list_urls |= spack.url.find_list_urls(aurl) # Add '/' to the end of the URL. Some web servers require this. additional_list_urls = set() for lurl in list_urls: if not lurl.endswith('/'): additional_list_urls.add(lurl + '/') list_urls |= additional_list_urls # Grab some web pages to scrape. pages, links = spider(list_urls, depth=list_depth, concurrency=concurrency) # Scrape them for archive URLs regexes = [] for aurl in archive_urls: # This creates a regex from the URL with a capture group for # the version part of the URL. The capture group is converted # to a generic wildcard, so we can use this to extract things # on a page that look like archive URLs. url_regex = spack.url.wildcard_version(aurl) # We'll be a bit more liberal and just look for the archive # part, not the full path. url_regex = os.path.basename(url_regex) # We need to add a / to the beginning of the regex to prevent # Spack from picking up similarly named packages like: # https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz # https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz # https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz # https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz url_regex = '/' + url_regex # We need to add a $ anchor to the end of the regex to prevent # Spack from picking up signature files like: # .asc # .md5 # .sha256 # .sig # However, SourceForge downloads still need to end in '/download'. url_regex += r'(\/download)?' # PyPI adds #sha256=... to the end of the URL url_regex += '(#sha256=.*)?' url_regex += '$' regexes.append(url_regex) # Build a dict version -> URL from any links that match the wildcards. # Walk through archive_url links first. # Any conflicting versions will be overwritten by the list_url links. versions = {} matched = set() for url in archive_urls + sorted(links): url = convert_to_posix_path(url) if any(re.search(r, url) for r in regexes): try: ver = spack.url.parse_version(url) if ver in matched: continue versions[ver] = url # prevent this version from getting overwritten if url in archive_urls: matched.add(ver) elif reference_package is not None: if url == reference_package.url_for_version(ver): matched.add(ver) else: extrapolated_urls = [ spack.url.substitute_version(u, ver) for u in archive_urls ] if url in extrapolated_urls: matched.add(ver) except spack.url.UndetectableVersionError: continue return versions
def make_installer(parser, args): """ Use CMake to generate WIX installer in newly created build directory """ if sys.platform == 'win32': output_dir = args.output_dir cmake_spec = Spec('cmake') cmake_spec.concretize() cmake_path = os.path.join(cmake_spec.prefix, "bin", "cmake.exe") cpack_path = os.path.join(cmake_spec.prefix, "bin", "cpack.exe") spack_source = args.spack_source git_verbosity = "" if args.git_verbosity: git_verbosity = "/" + args.git_verbosity if spack_source: if not os.path.exists(spack_source): print("%s does not exist" % spack_source) return else: if not os.path.isabs(spack_source): spack_source = posixpath.abspath(spack_source) spack_source = convert_to_posix_path(spack_source) spack_version = args.spack_version here = os.path.dirname(os.path.abspath(__file__)) source_dir = os.path.join(here, "installer") posix_root = convert_to_posix_path(spack.paths.spack_root) spack_license = posixpath.join(posix_root, "LICENSE-APACHE") rtf_spack_license = txt_to_rtf(spack_license) spack_license = posixpath.join(source_dir, "LICENSE.rtf") with open(spack_license, 'w') as rtf_license: written = rtf_license.write(rtf_spack_license) if written == 0: raise RuntimeError( "Failed to generate properly formatted license file") spack_logo = posixpath.join(posix_root, "share/spack/logo/favicon.ico") try: spack.util.executable.Executable(cmake_path)( '-S', source_dir, '-B', output_dir, '-DSPACK_VERSION=%s' % spack_version, '-DSPACK_SOURCE=%s' % spack_source, '-DSPACK_LICENSE=%s' % spack_license, '-DSPACK_LOGO=%s' % spack_logo, '-DSPACK_GIT_VERBOSITY=%s' % git_verbosity) except spack.util.executable.ProcessError: print("Failed to generate installer") return spack.util.executable.ProcessError.returncode try: spack.util.executable.Executable(cpack_path)( "--config", "%s/CPackConfig.cmake" % output_dir, "-B", "%s/" % output_dir) except spack.util.executable.ProcessError: print("Failed to generate installer") return spack.util.executable.ProcessError.returncode try: spack.util.executable.Executable( os.environ.get('WIX') + '/bin/candle.exe')( '-ext', 'WixBalExtension', '%s/bundle.wxs' % output_dir, '-out', '%s/bundle.wixobj' % output_dir) except spack.util.executable.ProcessError: print("Failed to generate installer chain") return spack.util.executable.ProcessError.returncode try: spack.util.executable.Executable( os.environ.get('WIX') + "/bin/light.exe")("-sw1134", "-ext", "WixBalExtension", "%s/bundle.wixobj" % output_dir, '-out', '%s/Spack.exe' % output_dir) except spack.util.executable.ProcessError: print("Failed to generate installer chain") return spack.util.executable.ProcessError.returncode print("Successfully generated Spack.exe in %s" % (output_dir)) else: print( 'The make-installer command is currently only supported on Windows.' )