def download_tarball(spec):
"""
Download binary tarball for given package into stage area
Return True if successful
"""
if not spack.mirror.MirrorCollection():
tty.die("Please add a spack mirror to allow " +
"download of pre-compiled packages.")
tarball = tarball_path_name(spec, '.spack')
for mirror in spack.mirror.MirrorCollection().values():
url = url_util.join(mirror.fetch_url, _build_cache_relative_path,
tarball)
# stage the tarball into standard place
stage = Stage(url, name="build_cache", keep=True)
stage.create()
try:
stage.fetch()
return stage.save_filename
except fs.FetchError:
continue
return None
def stage():
"""Creates a stage with the directory structure for the tests."""
s = Stage('link-tree-test')
s.create()
with working_dir(s.path):
touchp('source/1')
touchp('source/a/b/2')
touchp('source/a/b/3')
touchp('source/c/4')
touchp('source/c/d/5')
touchp('source/c/d/6')
touchp('source/c/d/e/7')
yield s
s.destroy()
def stage():
"""Creates a stage with the directory structure for the tests."""
s = Stage('link-tree-test')
s.create()
with working_dir(s.path):
touchp('source/1')
touchp('source/a/b/2')
touchp('source/a/b/3')
touchp('source/c/4')
touchp('source/c/d/5')
touchp('source/c/d/6')
touchp('source/c/d/e/7')
yield s
s.destroy()
def buildcache_sync(args):
""" Syncs binaries (and associated metadata) from one mirror to another.
Requires an active environment in order to know which specs to sync.
Args:
src (str): Source mirror URL
dest (str): Destination mirror URL
"""
# Figure out the source mirror
source_location = None
if args.src_directory:
source_location = args.src_directory
scheme = url_util.parse(source_location, scheme='<missing>').scheme
if scheme != '<missing>':
raise ValueError(
'"--src-directory" expected a local path; got a URL, instead')
# Ensure that the mirror lookup does not mistake this for named mirror
source_location = 'file://' + source_location
elif args.src_mirror_name:
source_location = args.src_mirror_name
result = spack.mirror.MirrorCollection().lookup(source_location)
if result.name == "<unnamed>":
raise ValueError('no configured mirror named "{name}"'.format(
name=source_location))
elif args.src_mirror_url:
source_location = args.src_mirror_url
scheme = url_util.parse(source_location, scheme='<missing>').scheme
if scheme == '<missing>':
raise ValueError(
'"{url}" is not a valid URL'.format(url=source_location))
src_mirror = spack.mirror.MirrorCollection().lookup(source_location)
src_mirror_url = url_util.format(src_mirror.fetch_url)
# Figure out the destination mirror
dest_location = None
if args.dest_directory:
dest_location = args.dest_directory
scheme = url_util.parse(dest_location, scheme='<missing>').scheme
if scheme != '<missing>':
raise ValueError(
'"--dest-directory" expected a local path; got a URL, instead')
# Ensure that the mirror lookup does not mistake this for named mirror
dest_location = 'file://' + dest_location
elif args.dest_mirror_name:
dest_location = args.dest_mirror_name
result = spack.mirror.MirrorCollection().lookup(dest_location)
if result.name == "<unnamed>":
raise ValueError('no configured mirror named "{name}"'.format(
name=dest_location))
elif args.dest_mirror_url:
dest_location = args.dest_mirror_url
scheme = url_util.parse(dest_location, scheme='<missing>').scheme
if scheme == '<missing>':
raise ValueError(
'"{url}" is not a valid URL'.format(url=dest_location))
dest_mirror = spack.mirror.MirrorCollection().lookup(dest_location)
dest_mirror_url = url_util.format(dest_mirror.fetch_url)
# Get the active environment
env = spack.cmd.require_active_env(cmd_name='buildcache sync')
tty.msg('Syncing environment buildcache files from {0} to {1}'.format(
src_mirror_url, dest_mirror_url))
build_cache_dir = bindist.build_cache_relative_path()
buildcache_rel_paths = []
tty.debug('Syncing the following specs:')
for s in env.all_specs():
tty.debug(' {0}{1}: {2}'.format('* ' if s in env.roots() else ' ',
s.name, s.dag_hash()))
buildcache_rel_paths.extend([
os.path.join(build_cache_dir,
bindist.tarball_path_name(s, '.spack')),
os.path.join(build_cache_dir,
bindist.tarball_name(s, '.spec.yaml')),
os.path.join(build_cache_dir, bindist.tarball_name(s, '.cdashid'))
])
tmpdir = tempfile.mkdtemp()
try:
for rel_path in buildcache_rel_paths:
src_url = url_util.join(src_mirror_url, rel_path)
local_path = os.path.join(tmpdir, rel_path)
dest_url = url_util.join(dest_mirror_url, rel_path)
tty.debug('Copying {0} to {1} via {2}'.format(
src_url, dest_url, local_path))
stage = Stage(src_url,
name="temporary_file",
path=os.path.dirname(local_path),
keep=True)
try:
stage.create()
stage.fetch()
web_util.push_to_url(local_path, dest_url, keep_original=True)
except fs.FetchError as e:
tty.debug(
'spack buildcache unable to sync {0}'.format(rel_path))
tty.debug(e)
finally:
stage.destroy()
finally:
shutil.rmtree(tmpdir)
class LinkTreeTest(unittest.TestCase):
"""Tests Spack's LinkTree class."""
def setUp(self):
self.stage = Stage('link-tree-test')
self.stage.create()
with working_dir(self.stage.path):
touchp('source/1')
touchp('source/a/b/2')
touchp('source/a/b/3')
touchp('source/c/4')
touchp('source/c/d/5')
touchp('source/c/d/6')
touchp('source/c/d/e/7')
source_path = os.path.join(self.stage.path, 'source')
self.link_tree = LinkTree(source_path)
def tearDown(self):
self.stage.destroy()
def check_file_link(self, filename):
self.assertTrue(os.path.isfile(filename))
self.assertTrue(os.path.islink(filename))
def check_dir(self, filename):
self.assertTrue(os.path.isdir(filename))
def test_merge_to_new_directory(self):
with working_dir(self.stage.path):
self.link_tree.merge('dest')
self.check_file_link('dest/1')
self.check_file_link('dest/a/b/2')
self.check_file_link('dest/a/b/3')
self.check_file_link('dest/c/4')
self.check_file_link('dest/c/d/5')
self.check_file_link('dest/c/d/6')
self.check_file_link('dest/c/d/e/7')
self.link_tree.unmerge('dest')
self.assertFalse(os.path.exists('dest'))
def test_merge_to_existing_directory(self):
with working_dir(self.stage.path):
touchp('dest/x')
touchp('dest/a/b/y')
self.link_tree.merge('dest')
self.check_file_link('dest/1')
self.check_file_link('dest/a/b/2')
self.check_file_link('dest/a/b/3')
self.check_file_link('dest/c/4')
self.check_file_link('dest/c/d/5')
self.check_file_link('dest/c/d/6')
self.check_file_link('dest/c/d/e/7')
self.assertTrue(os.path.isfile('dest/x'))
self.assertTrue(os.path.isfile('dest/a/b/y'))
self.link_tree.unmerge('dest')
self.assertTrue(os.path.isfile('dest/x'))
self.assertTrue(os.path.isfile('dest/a/b/y'))
self.assertFalse(os.path.isfile('dest/1'))
self.assertFalse(os.path.isfile('dest/a/b/2'))
self.assertFalse(os.path.isfile('dest/a/b/3'))
self.assertFalse(os.path.isfile('dest/c/4'))
self.assertFalse(os.path.isfile('dest/c/d/5'))
self.assertFalse(os.path.isfile('dest/c/d/6'))
self.assertFalse(os.path.isfile('dest/c/d/e/7'))
def test_merge_with_empty_directories(self):
with working_dir(self.stage.path):
mkdirp('dest/f/g')
mkdirp('dest/a/b/h')
self.link_tree.merge('dest')
self.link_tree.unmerge('dest')
self.assertFalse(os.path.exists('dest/1'))
self.assertFalse(os.path.exists('dest/a/b/2'))
self.assertFalse(os.path.exists('dest/a/b/3'))
self.assertFalse(os.path.exists('dest/c/4'))
self.assertFalse(os.path.exists('dest/c/d/5'))
self.assertFalse(os.path.exists('dest/c/d/6'))
self.assertFalse(os.path.exists('dest/c/d/e/7'))
self.assertTrue(os.path.isdir('dest/a/b/h'))
self.assertTrue(os.path.isdir('dest/f/g'))
def test_ignore(self):
with working_dir(self.stage.path):
touchp('source/.spec')
touchp('dest/.spec')
self.link_tree.merge('dest', ignore=lambda x: x == '.spec')
self.link_tree.unmerge('dest', ignore=lambda x: x == '.spec')
self.assertFalse(os.path.exists('dest/1'))
self.assertFalse(os.path.exists('dest/a'))
self.assertFalse(os.path.exists('dest/c'))
self.assertTrue(os.path.isfile('source/.spec'))
self.assertTrue(os.path.isfile('dest/.spec'))
class LinkTreeTest(unittest.TestCase):
"""Tests Spack's LinkTree class."""
def setUp(self):
self.stage = Stage('link-tree-test')
self.stage.create()
with working_dir(self.stage.path):
touchp('source/1')
touchp('source/a/b/2')
touchp('source/a/b/3')
touchp('source/c/4')
touchp('source/c/d/5')
touchp('source/c/d/6')
touchp('source/c/d/e/7')
source_path = os.path.join(self.stage.path, 'source')
self.link_tree = LinkTree(source_path)
def tearDown(self):
self.stage.destroy()
def check_file_link(self, filename):
self.assertTrue(os.path.isfile(filename))
self.assertTrue(os.path.islink(filename))
def check_dir(self, filename):
self.assertTrue(os.path.isdir(filename))
def test_merge_to_new_directory(self):
with working_dir(self.stage.path):
self.link_tree.merge('dest')
self.check_file_link('dest/1')
self.check_file_link('dest/a/b/2')
self.check_file_link('dest/a/b/3')
self.check_file_link('dest/c/4')
self.check_file_link('dest/c/d/5')
self.check_file_link('dest/c/d/6')
self.check_file_link('dest/c/d/e/7')
self.link_tree.unmerge('dest')
self.assertFalse(os.path.exists('dest'))
def test_merge_to_existing_directory(self):
with working_dir(self.stage.path):
touchp('dest/x')
touchp('dest/a/b/y')
self.link_tree.merge('dest')
self.check_file_link('dest/1')
self.check_file_link('dest/a/b/2')
self.check_file_link('dest/a/b/3')
self.check_file_link('dest/c/4')
self.check_file_link('dest/c/d/5')
self.check_file_link('dest/c/d/6')
self.check_file_link('dest/c/d/e/7')
self.assertTrue(os.path.isfile('dest/x'))
self.assertTrue(os.path.isfile('dest/a/b/y'))
self.link_tree.unmerge('dest')
self.assertTrue(os.path.isfile('dest/x'))
self.assertTrue(os.path.isfile('dest/a/b/y'))
self.assertFalse(os.path.isfile('dest/1'))
self.assertFalse(os.path.isfile('dest/a/b/2'))
self.assertFalse(os.path.isfile('dest/a/b/3'))
self.assertFalse(os.path.isfile('dest/c/4'))
self.assertFalse(os.path.isfile('dest/c/d/5'))
self.assertFalse(os.path.isfile('dest/c/d/6'))
self.assertFalse(os.path.isfile('dest/c/d/e/7'))
def test_merge_with_empty_directories(self):
with working_dir(self.stage.path):
mkdirp('dest/f/g')
mkdirp('dest/a/b/h')
self.link_tree.merge('dest')
self.link_tree.unmerge('dest')
self.assertFalse(os.path.exists('dest/1'))
self.assertFalse(os.path.exists('dest/a/b/2'))
self.assertFalse(os.path.exists('dest/a/b/3'))
self.assertFalse(os.path.exists('dest/c/4'))
self.assertFalse(os.path.exists('dest/c/d/5'))
self.assertFalse(os.path.exists('dest/c/d/6'))
self.assertFalse(os.path.exists('dest/c/d/e/7'))
self.assertTrue(os.path.isdir('dest/a/b/h'))
self.assertTrue(os.path.isdir('dest/f/g'))
def test_ignore(self):
with working_dir(self.stage.path):
touchp('source/.spec')
touchp('dest/.spec')
self.link_tree.merge('dest', ignore=lambda x: x == '.spec')
self.link_tree.unmerge('dest', ignore=lambda x: x == '.spec')
self.assertFalse(os.path.exists('dest/1'))
self.assertFalse(os.path.exists('dest/a'))
self.assertFalse(os.path.exists('dest/c'))
self.assertTrue(os.path.isfile('source/.spec'))
self.assertTrue(os.path.isfile('dest/.spec'))
