def issue_request(self, request, retry=True):
"""
Given a prepared request, issue it.
If we get an error, die. If
there are times when we don't want to exit on error (but instead
disable using the monitoring service) we could add that here.
"""
try:
response = urlopen(request)
except URLError as e:
# If we have an authorization request, retry once with auth
if hasattr(e, "code") and e.code == 401 and retry:
if self.authenticate_request(e):
request = self.prepare_request(
e.url,
sjson.load(request.data.decode('utf-8')),
self.headers
)
return self.issue_request(request, False)
# Handle permanent re-directs!
elif hasattr(e, "code") and e.code == 308:
location = e.headers.get('Location')
request_data = None
if request.data:
request_data = sjson.load(request.data.decode('utf-8'))[0]
if location:
request = self.prepare_request(
location,
request_data,
self.headers
)
return self.issue_request(request, True)
# Otherwise, relay the message and exit on error
msg = ""
if hasattr(e, 'reason'):
msg = e.reason
elif hasattr(e, 'code'):
msg = e.code
# If we can parse the message, try it
try:
msg += "\n%s" % e.read().decode("utf8", 'ignore')
except Exception:
pass
if self.allow_fail:
tty.warning("Request to %s was not successful, but continuing." % e.url)
return
tty.die(msg)
return response
def check_file_manifest(filename):
dirname = os.path.dirname(filename)
results = VerificationResults()
while spack.store.layout.metadata_dir not in os.listdir(dirname):
if dirname == os.path.sep:
results.add_error(filename, 'not owned by any package')
return results
dirname = os.path.dirname(dirname)
manifest_file = os.path.join(dirname, spack.store.layout.metadata_dir,
spack.store.layout.manifest_file_name)
if not os.path.exists(manifest_file):
results.add_error(filename, "manifest missing")
return results
try:
with open(manifest_file, 'r') as f:
manifest = sjson.load(f)
except Exception:
results.add_error(filename, "manifest corrupted")
return results
if filename in manifest:
results += check_entry(filename, manifest[filename])
else:
results.add_error(filename, 'not owned by any package')
return results
def test_environment_analyzer(tmpdir, mock_fetch,
install_mockery_mutable_config):
"""
test the environment variables analyzer.
"""
install('libdwarf')
output_file = _run_analyzer("environment_variables", "libdwarf", tmpdir)
with open(output_file, 'r') as fd:
content = sjson.load(fd.read())
# Check a few expected keys
for key in ['SPACK_CC', 'SPACK_COMPILER_SPEC', 'SPACK_ENV_PATH']:
assert key in content
# The analyzer should return no result if the output file does not exist.
spec = Spec('libdwarf').concretized()
env_file = os.path.join(spec.package.prefix, '.spack',
'spack-build-env.txt')
assert os.path.exists(env_file)
os.remove(env_file)
analyzer = spack.analyzers.get_analyzer("environment_variables")
analyzer_dir = tmpdir.join('analyzers')
result = analyzer(spec, analyzer_dir).run()
assert "environment_variables" in result
assert not result['environment_variables']
def get_build_id(self, spec, return_response=False, spec_exists=True):
"""
Retrieve a build id, either in the local cache, or query the server.
"""
full_hash = spec.full_hash()
if full_hash in self.build_ids:
return self.build_ids[full_hash]
# Prepare build environment data (including spack version)
data = self.build_environment.copy()
data['full_hash'] = full_hash
# If the build should be tagged, add it
if self.tags:
data['tags'] = self.tags
# If we allow the spec to not exist (meaning we create it) we need to
# include the full specfile here
if not spec_exists:
meta_dir = os.path.dirname(spec.package.install_log_path)
spec_file = os.path.join(meta_dir, "spec.json")
if os.path.exists(spec_file):
data['spec'] = sjson.load(read_file(spec_file))
else:
spec_file = os.path.join(meta_dir, "spec.yaml")
data['spec'] = syaml.load(read_file(spec_file))
if self.save_local:
return self.get_local_build_id(data, full_hash, return_response)
return self.get_server_build_id(data, full_hash, return_response)
def load_data(self):
"""
Load data if the path already exists.
"""
if os.path.isfile(self.cache_path):
with spack.caches.misc_cache.read_transaction(self.cache_key) as cache_file:
self.data = sjson.load(cache_file)
def read_json(filename):
"""
Read a file and load into json, if it exists. Otherwise return None.
"""
if not os.path.exists(filename):
return
return sjson.load(read_file(filename))
def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages):
"""Test with and without the --first option"""
install_cmd('mpileaks')
# Only one version of mpileaks will work
diff_cmd('mpileaks', 'mpileaks')
# 2 specs are required for a diff
with pytest.raises(spack.main.SpackCommandError):
diff_cmd('mpileaks')
with pytest.raises(spack.main.SpackCommandError):
diff_cmd('mpileaks', 'mpileaks', 'mpileaks')
# Ensure they are the same
assert "No differences" in diff_cmd('mpileaks', 'mpileaks')
output = diff_cmd('--json', 'mpileaks', 'mpileaks')
result = sjson.load(output)
assert len(result['a_not_b']) == 0
assert len(result['b_not_a']) == 0
assert 'mpileaks' in result['a_name']
assert 'mpileaks' in result['b_name']
assert "intersect" in result and len(result['intersect']) > 50
# After we install another version, it should ask us to disambiguate
install_cmd('mpileaks+debug')
# There are two versions of mpileaks
with pytest.raises(spack.main.SpackCommandError):
diff_cmd('mpileaks', 'mpileaks+debug')
# But if we tell it to use the first, it won't try to disambiguate
assert "variant" in diff_cmd('--first', 'mpileaks', 'mpileaks+debug')
# This matches them exactly
debug_hash = find_cmd('--format', '{hash}', 'mpileaks+debug').strip()
no_debug_hashes = find_cmd('--format', '{hash}', 'mpileaks~debug')
no_debug_hash = no_debug_hashes.split()[0]
output = diff_cmd("--json", "mpileaks/{0}".format(debug_hash),
"mpileaks/{0}".format(no_debug_hash))
result = sjson.load(output)
assert len(result['a_not_b']) == 1
assert len(result['b_not_a']) == 1
assert result['a_not_b'][0] == ['variant_value', 'mpileaks debug True']
assert result['b_not_a'][0] == ['variant_value', 'mpileaks debug False']
def test_load_first(install_mockery, mock_fetch, mock_archive, mock_packages):
"""Test with and without the --first option"""
install('mpileaks')
# Only one version of mpileaks will work
diff('mpileaks', 'mpileaks')
# 2 specs are required for a diff
with pytest.raises(spack.main.SpackCommandError):
diff('mpileaks')
with pytest.raises(spack.main.SpackCommandError):
diff('mpileaks', 'mpileaks', 'mpileaks')
# Ensure they are the same
assert "No differences" in diff('mpileaks', 'mpileaks')
output = diff('--json', 'mpileaks', 'mpileaks')
result = sjson.load(output)
assert len(result['a_not_b']) == 0
assert len(result['b_not_a']) == 0
assert 'mpileaks' in result['a_name']
assert 'mpileaks' in result['b_name']
assert "intersect" in result and len(result['intersect']) > 50
# After we install another version, it should ask us to disambiguate
install('mpileaks+debug')
# There are two versions of mpileaks
with pytest.raises(spack.main.SpackCommandError):
diff('mpileaks', 'mpileaks+debug')
# But if we tell it to use the first, it won't try to disambiguate
assert "variant" in diff('--first', 'mpileaks', 'mpileaks+debug')
# This matches them exactly
output = diff("--json", "[email protected]/ysubb76", "[email protected]/ft5qff3")
result = sjson.load(output)
assert len(result['a_not_b']) == 1
assert len(result['b_not_a']) == 1
assert result['a_not_b'][0] == [
'variant_value', 'mpileaks debug bool(False)'
]
assert result['b_not_a'][0] == [
'variant_value', 'mpileaks debug bool(True)'
]
def from_file(filename):
try:
with open(filename, 'r') as f:
data = sjson.load(f)
return TestSuite.from_dict(data)
except Exception as e:
tty.debug(e)
raise sjson.SpackJSONError("error parsing JSON TestSuite:", str(e))
def from_json(stream):
d = sjson.load(stream)
r = TagIndex()
for tag, list in d['tags'].items():
r[tag].extend(list)
return r
def from_json(stream, name=None):
try:
d = sjson.load(stream)
return Mirror.from_dict(d, name)
except Exception as e:
raise six.raise_from(
sjson.SpackJSONError("error parsing JSON mirror:", str(e)),
e,
)
def get_keys(install=False, trust=False, force=False, mirrors=None):
"""Get pgp public keys available on mirror with suffix .pub
"""
mirror_collection = (mirrors or spack.mirror.MirrorCollection())
if not mirror_collection:
tty.die("Please add a spack mirror to allow " +
"download of build caches.")
for mirror in mirror_collection.values():
fetch_url = mirror.fetch_url
keys_url = url_util.join(fetch_url, _build_cache_relative_path,
_build_cache_keys_relative_path)
keys_index = url_util.join(keys_url, 'index.json')
tty.debug('Finding public keys in {0}'.format(
url_util.format(fetch_url)))
try:
_, _, json_file = web_util.read_from_url(keys_index)
json_index = sjson.load(codecs.getreader('utf-8')(json_file))
except (URLError, web_util.SpackWebError) as url_err:
if web_util.url_exists(keys_index):
err_msg = [
'Unable to find public keys in {0},',
' caught exception attempting to read from {1}.',
]
tty.error(''.join(err_msg).format(url_util.format(fetch_url),
url_util.format(keys_index)))
tty.debug(url_err)
continue
for fingerprint, key_attributes in json_index['keys'].items():
link = os.path.join(keys_url, fingerprint + '.pub')
with Stage(link, name="build_cache", keep=True) as stage:
if os.path.exists(stage.save_filename) and force:
os.remove(stage.save_filename)
if not os.path.exists(stage.save_filename):
try:
stage.fetch()
except fs.FetchError:
continue
tty.debug('Found key {0}'.format(fingerprint))
if install:
if trust:
spack.util.gpg.trust(stage.save_filename)
tty.debug('Added this key to trusted keys.')
else:
tty.debug('Will not add this key to trusted keys.'
'Use -t to install all downloaded keys')
def from_file(filename):
try:
with open(filename, 'r') as f:
data = sjson.load(f)
test_suite = TestSuite.from_dict(data)
content_hash = os.path.basename(os.path.dirname(filename))
test_suite._hash = content_hash
return test_suite
except Exception as e:
tty.debug(e)
raise sjson.SpackJSONError("error parsing JSON TestSuite:", str(e))
def from_json(stream):
d = sjson.load(stream)
if not isinstance(d, dict):
raise TagIndexError("TagIndex data was not a dict.")
if 'tags' not in d:
raise TagIndexError("TagIndex data does not start with 'tags'")
r = TagIndex()
for tag, packages in d['tags'].items():
r[tag].extend(packages)
return r
def needs_file(spec, file):
# convert the file we want to remove to a source in this spec
projection = self.get_projection_for_spec(spec)
relative_path = os.path.relpath(file, projection)
test_path = os.path.join(spec.prefix, relative_path)
# check if this spec owns a file of that name (through the
# manifest in the metadata dir, which we have in the view).
manifest_file = os.path.join(self.get_path_meta_folder(spec),
spack.store.layout.manifest_file_name)
try:
with open(manifest_file, 'r') as f:
manifest = s_json.load(f)
except (OSError, IOError):
# if we can't load it, assume it doesn't know about the file.
manifest = {}
return test_path in manifest
def from_json(stream):
data = sjson.load(stream)
if not isinstance(data, dict):
raise ProviderIndexError("JSON ProviderIndex data was not a dict.")
if 'provider_index' not in data:
raise ProviderIndexError(
"YAML ProviderIndex does not start with 'provider_index'")
index = ProviderIndex()
providers = data['provider_index']['providers']
index.providers = _transform(
providers, lambda vpkg, plist:
(spack.spec.Spec.from_node_dict(vpkg),
set(spack.spec.Spec.from_node_dict(p) for p in plist)))
return index
def do_request(self, endpoint, data=None, headers=None, url=None):
"""
Do the actual request.
If data is provided, it is POST, otherwise GET.
If an entire URL is provided, don't use the endpoint
"""
request = self.prepare_request(endpoint, data, headers)
# If we have an authorization error, we retry with
response = self.issue_request(request)
# A 200/201 response incidates success
if response.code in [200, 201]:
return sjson.load(response.read().decode('utf-8'))
return response
def _load_distutil_vars(self):
# We update and keep the cache unchanged only if the package is
# installed.
if not self._distutil_vars and self.installed:
try:
input_filename = join_path(
spack.store.layout.metadata_path(self.spec),
Python._DISTUTIL_CACHE_FILENAME)
if os.path.isfile(input_filename):
with open(input_filename) as input_file:
self._distutil_vars = sjson.load(input_file)
except Exception:
pass
if not self._distutil_vars:
self._distutil_vars = {}
return self._distutil_vars
def _load_distutil_vars(self):
# We update and keep the cache unchanged only if the package is
# installed.
if not self._distutil_vars and self.installed:
try:
input_filename = join_path(
spack.store.layout.metadata_path(self.spec),
Python._DISTUTIL_CACHE_FILENAME)
if os.path.isfile(input_filename):
with open(input_filename) as input_file:
self._distutil_vars = sjson.load(input_file)
except:
pass
if not self._distutil_vars:
self._distutil_vars = {}
return self._distutil_vars
def test_single_file_verify_cmd(tmpdir):
# Test the verify command interface to verifying a single file.
filedir = os.path.join(str(tmpdir), 'a', 'b', 'c', 'd')
filepath = os.path.join(filedir, 'file')
metadir = os.path.join(str(tmpdir), spack.store.layout.metadata_dir)
fs.mkdirp(filedir)
fs.mkdirp(metadir)
with open(filepath, 'w') as f:
f.write("I'm a file")
data = spack.verify.create_manifest_entry(filepath)
manifest_file = os.path.join(metadir,
spack.store.layout.manifest_file_name)
with open(manifest_file, 'w') as f:
sjson.dump({filepath: data}, f)
results = verify('-f', filepath, fail_on_error=False)
print(results)
assert not results
os.utime(filepath, (0, 0))
with open(filepath, 'w') as f:
f.write("I changed.")
results = verify('-f', filepath, fail_on_error=False)
expected = ['hash']
mtime = os.stat(filepath).st_mtime
if mtime != data['time']:
expected.append('mtime')
assert results
assert filepath in results
assert all(x in results for x in expected)
results = verify('-fj', filepath, fail_on_error=False)
res = sjson.load(results)
assert len(res) == 1
errors = res.pop(filepath)
assert sorted(errors) == sorted(expected)
def test_installfiles_analyzer(tmpdir, mock_fetch,
install_mockery_mutable_config):
"""
test the install files analyzer
"""
install('libdwarf')
output_file = _run_analyzer("install_files", "libdwarf", tmpdir)
# Ensure it's the correct content
with open(output_file, 'r') as fd:
content = sjson.load(fd.read())
basenames = set()
for key, attrs in content.items():
basenames.add(os.path.basename(key))
# Check for a few expected files
for key in ['.spack', 'libdwarf', 'packages', 'repo.yaml', 'repos']:
assert key in basenames
def test_blame_json(mock_packages):
"""Ensure that we can output json as a blame."""
with working_dir(spack.paths.prefix):
out = blame('--json', 'mpich')
# Test loading the json, and top level keys
loaded = sjson.load(out)
assert "authors" in out
assert "totals" in out
# Authors should be a list
assert len(loaded['authors']) > 0
# Each of authors and totals has these shared keys
keys = ["last_commit", "lines", "percentage"]
for key in keys:
assert key in loaded['totals']
# But authors is a list of multiple
for key in keys + ["author", "email"]:
assert key in loaded['authors'][0]
def from_json(stream):
"""Construct a provider index from its JSON representation.
Args:
stream: stream where to read from the JSON data
"""
data = sjson.load(stream)
if not isinstance(data, dict):
raise ProviderIndexError("JSON ProviderIndex data was not a dict.")
if 'provider_index' not in data:
raise ProviderIndexError(
"YAML ProviderIndex does not start with 'provider_index'")
index = ProviderIndex()
providers = data['provider_index']['providers']
index.providers = _transform(
providers, lambda vpkg, plist:
(spack.spec.Spec.from_node_dict(vpkg),
set(spack.spec.Spec.from_node_dict(p) for p in plist)))
return index
def test_single_spec_verify_cmd(tmpdir, mock_packages, mock_archive,
mock_fetch, config, install_mockery):
# Test the verify command interface to verify a single spec
install('libelf')
s = spack.spec.Spec('libelf').concretized()
prefix = s.prefix
hash = s.dag_hash()
results = verify('/%s' % hash, fail_on_error=False)
assert not results
new_file = os.path.join(prefix, 'new_file_for_verify_test')
with open(new_file, 'w') as f:
f.write('New file')
results = verify('/%s' % hash, fail_on_error=False)
assert new_file in results
assert 'added' in results
results = verify('-j', '/%s' % hash, fail_on_error=False)
res = sjson.load(results)
assert len(res) == 1
assert res[new_file] == ['added']
def test_install_times(install_mockery, mock_fetch, mutable_mock_repo):
"""Test install times added."""
spec = Spec('dev-build-test-install-phases')
spec.concretize()
pkg = spec.package
pkg.do_install()
# Ensure dependency directory exists after the installation.
install_times = os.path.join(pkg.prefix, ".spack", 'install_times.json')
assert os.path.isfile(install_times)
# Ensure the phases are included
with open(install_times, 'r') as timefile:
times = sjson.load(timefile.read())
# The order should be maintained
phases = [x['name'] for x in times['phases']]
total = sum([x['seconds'] for x in times['phases']])
for name in ['one', 'two', 'three', 'install']:
assert name in phases
# Give a generous difference threshold
assert abs(total - times['total']['seconds']) < 5
def issue_request(self, request, retry=True):
"""
Given a prepared request, issue it.
If we get an error, die. If
there are times when we don't want to exit on error (but instead
disable using the monitoring service) we could add that here.
"""
try:
response = urlopen(request)
except URLError as e:
# If we have an authorization request, retry once with auth
if hasattr(e, "code") and e.code == 401 and retry:
if self.authenticate_request(e):
request = self.prepare_request(
e.url,
sjson.load(request.data.decode('utf-8')),
self.headers
)
return self.issue_request(request, False)
# Otherwise, relay the message and exit on error
msg = ""
if hasattr(e, 'reason'):
msg = e.reason
elif hasattr(e, 'code'):
msg = e.code
if self.allow_fail:
tty.warning("Request to %s was not successful, but continuing." % e.url)
return
tty.die(msg)
return response
def test_read_old_lock_creates_backup(tmpdir):
"""When reading a version-1 lockfile, make sure that a backup of that file
is created.
"""
y = MockPackage('y', [], [])
mock_repo = MockPackageMultiRepo([y])
with spack.repo.swap(mock_repo):
y = Spec('y')
y.concretize()
test_lockfile_dict = create_v1_lockfile_dict([y], [y])
env_root = tmpdir.mkdir('test-root')
test_lockfile_path = str(env_root.join(ev.lockfile_name))
with open(test_lockfile_path, 'w') as f:
sjson.dump(test_lockfile_dict, stream=f)
e = ev.Environment(str(env_root))
assert os.path.exists(e._lock_backup_v1_path)
with open(e._lock_backup_v1_path, 'r') as backup_v1_file:
lockfile_dict_v1 = sjson.load(backup_v1_file)
# Make sure that the backup file follows the v1 hash scheme
assert y.dag_hash() in lockfile_dict_v1['concrete_specs']
def _read_lockfile(self, file_or_json):
"""Read a lockfile from a file or from a raw string."""
lockfile_dict = sjson.load(file_or_json)
self._read_lockfile_dict(lockfile_dict)
def from_json(cls, stream):
return PatchCache(sjson.load(stream))
def from_json(stream, name=None):
d = sjson.load(stream)
return Mirror.from_dict(d, name)
def from_json(stream, name=None):
d = sjson.load(stream)
return MirrorCollection(d)
def check_spec_manifest(spec):
prefix = spec.prefix
results = VerificationResults()
manifest_file = os.path.join(prefix, spack.store.layout.metadata_dir,
spack.store.layout.manifest_file_name)
if not os.path.exists(manifest_file):
results.add_error(prefix, "manifest missing")
return results
try:
with open(manifest_file, 'r') as f:
manifest = sjson.load(f)
except Exception:
results.add_error(prefix, "manifest corrupted")
return results
# Get extensions active in spec
view = spack.filesystem_view.YamlFilesystemView(prefix, spack.store.layout)
active_exts = view.extensions_layout.extension_map(spec).values()
ext_file = ''
if active_exts:
# No point checking contents of this file as it is the only source of
# truth for that information.
ext_file = view.extensions_layout.extension_file_path(spec)
def is_extension_artifact(p):
if os.path.islink(p):
if any(os.readlink(p).startswith(e.prefix) for e in active_exts):
# This file is linked in by an extension. Belongs to extension
return True
elif os.path.isdir(p) and p not in manifest:
if all(
is_extension_artifact(os.path.join(p, f))
for f in os.listdir(p)):
return True
return False
for root, dirs, files in os.walk(prefix):
for entry in list(dirs + files):
path = os.path.join(root, entry)
# Do not check links from prefix to active extension
# TODO: make this stricter for non-linux systems that use symlink
# permissions
# Do not check directories that only exist for extensions
if is_extension_artifact(path):
continue
# Do not check manifest file. Can't store your own hash
# Nothing to check for ext_file
if path == manifest_file or path == ext_file:
continue
data = manifest.pop(path, {})
results += check_entry(path, data)
results += check_entry(prefix, manifest.pop(prefix, {}))
for path in manifest:
results.add_error(path, 'deleted')
return results