def download_tarball(spec): """ Download binary tarball for given package into stage area Return True if successful """ if not spack.mirror.MirrorCollection(): tty.die("Please add a spack mirror to allow " + "download of pre-compiled packages.") tarball = tarball_path_name(spec, '.spack') for mirror in spack.mirror.MirrorCollection().values(): url = url_util.join(mirror.fetch_url, _build_cache_relative_path, tarball) # stage the tarball into standard place stage = Stage(url, name="build_cache", keep=True) stage.create() try: stage.fetch() return stage.save_filename except fs.FetchError: continue return None
def stage(): """Creates a stage with the directory structure for the tests.""" s = Stage('link-tree-test') s.create() with working_dir(s.path): touchp('source/1') touchp('source/a/b/2') touchp('source/a/b/3') touchp('source/c/4') touchp('source/c/d/5') touchp('source/c/d/6') touchp('source/c/d/e/7') yield s s.destroy()
def buildcache_sync(args): """ Syncs binaries (and associated metadata) from one mirror to another. Requires an active environment in order to know which specs to sync. Args: src (str): Source mirror URL dest (str): Destination mirror URL """ # Figure out the source mirror source_location = None if args.src_directory: source_location = args.src_directory scheme = url_util.parse(source_location, scheme='<missing>').scheme if scheme != '<missing>': raise ValueError( '"--src-directory" expected a local path; got a URL, instead') # Ensure that the mirror lookup does not mistake this for named mirror source_location = 'file://' + source_location elif args.src_mirror_name: source_location = args.src_mirror_name result = spack.mirror.MirrorCollection().lookup(source_location) if result.name == "<unnamed>": raise ValueError('no configured mirror named "{name}"'.format( name=source_location)) elif args.src_mirror_url: source_location = args.src_mirror_url scheme = url_util.parse(source_location, scheme='<missing>').scheme if scheme == '<missing>': raise ValueError( '"{url}" is not a valid URL'.format(url=source_location)) src_mirror = spack.mirror.MirrorCollection().lookup(source_location) src_mirror_url = url_util.format(src_mirror.fetch_url) # Figure out the destination mirror dest_location = None if args.dest_directory: dest_location = args.dest_directory scheme = url_util.parse(dest_location, scheme='<missing>').scheme if scheme != '<missing>': raise ValueError( '"--dest-directory" expected a local path; got a URL, instead') # Ensure that the mirror lookup does not mistake this for named mirror dest_location = 'file://' + dest_location elif args.dest_mirror_name: dest_location = args.dest_mirror_name result = spack.mirror.MirrorCollection().lookup(dest_location) if result.name == "<unnamed>": raise ValueError('no configured mirror named "{name}"'.format( name=dest_location)) elif args.dest_mirror_url: dest_location = args.dest_mirror_url scheme = url_util.parse(dest_location, scheme='<missing>').scheme if scheme == '<missing>': raise ValueError( '"{url}" is not a valid URL'.format(url=dest_location)) dest_mirror = spack.mirror.MirrorCollection().lookup(dest_location) dest_mirror_url = url_util.format(dest_mirror.fetch_url) # Get the active environment env = spack.cmd.require_active_env(cmd_name='buildcache sync') tty.msg('Syncing environment buildcache files from {0} to {1}'.format( src_mirror_url, dest_mirror_url)) build_cache_dir = bindist.build_cache_relative_path() buildcache_rel_paths = [] tty.debug('Syncing the following specs:') for s in env.all_specs(): tty.debug(' {0}{1}: {2}'.format('* ' if s in env.roots() else ' ', s.name, s.dag_hash())) buildcache_rel_paths.extend([ os.path.join(build_cache_dir, bindist.tarball_path_name(s, '.spack')), os.path.join(build_cache_dir, bindist.tarball_name(s, '.spec.yaml')), os.path.join(build_cache_dir, bindist.tarball_name(s, '.cdashid')) ]) tmpdir = tempfile.mkdtemp() try: for rel_path in buildcache_rel_paths: src_url = url_util.join(src_mirror_url, rel_path) local_path = os.path.join(tmpdir, rel_path) dest_url = url_util.join(dest_mirror_url, rel_path) tty.debug('Copying {0} to {1} via {2}'.format( src_url, dest_url, local_path)) stage = Stage(src_url, name="temporary_file", path=os.path.dirname(local_path), keep=True) try: stage.create() stage.fetch() web_util.push_to_url(local_path, dest_url, keep_original=True) except fs.FetchError as e: tty.debug( 'spack buildcache unable to sync {0}'.format(rel_path)) tty.debug(e) finally: stage.destroy() finally: shutil.rmtree(tmpdir)
class LinkTreeTest(unittest.TestCase): """Tests Spack's LinkTree class.""" def setUp(self): self.stage = Stage('link-tree-test') self.stage.create() with working_dir(self.stage.path): touchp('source/1') touchp('source/a/b/2') touchp('source/a/b/3') touchp('source/c/4') touchp('source/c/d/5') touchp('source/c/d/6') touchp('source/c/d/e/7') source_path = os.path.join(self.stage.path, 'source') self.link_tree = LinkTree(source_path) def tearDown(self): self.stage.destroy() def check_file_link(self, filename): self.assertTrue(os.path.isfile(filename)) self.assertTrue(os.path.islink(filename)) def check_dir(self, filename): self.assertTrue(os.path.isdir(filename)) def test_merge_to_new_directory(self): with working_dir(self.stage.path): self.link_tree.merge('dest') self.check_file_link('dest/1') self.check_file_link('dest/a/b/2') self.check_file_link('dest/a/b/3') self.check_file_link('dest/c/4') self.check_file_link('dest/c/d/5') self.check_file_link('dest/c/d/6') self.check_file_link('dest/c/d/e/7') self.link_tree.unmerge('dest') self.assertFalse(os.path.exists('dest')) def test_merge_to_existing_directory(self): with working_dir(self.stage.path): touchp('dest/x') touchp('dest/a/b/y') self.link_tree.merge('dest') self.check_file_link('dest/1') self.check_file_link('dest/a/b/2') self.check_file_link('dest/a/b/3') self.check_file_link('dest/c/4') self.check_file_link('dest/c/d/5') self.check_file_link('dest/c/d/6') self.check_file_link('dest/c/d/e/7') self.assertTrue(os.path.isfile('dest/x')) self.assertTrue(os.path.isfile('dest/a/b/y')) self.link_tree.unmerge('dest') self.assertTrue(os.path.isfile('dest/x')) self.assertTrue(os.path.isfile('dest/a/b/y')) self.assertFalse(os.path.isfile('dest/1')) self.assertFalse(os.path.isfile('dest/a/b/2')) self.assertFalse(os.path.isfile('dest/a/b/3')) self.assertFalse(os.path.isfile('dest/c/4')) self.assertFalse(os.path.isfile('dest/c/d/5')) self.assertFalse(os.path.isfile('dest/c/d/6')) self.assertFalse(os.path.isfile('dest/c/d/e/7')) def test_merge_with_empty_directories(self): with working_dir(self.stage.path): mkdirp('dest/f/g') mkdirp('dest/a/b/h') self.link_tree.merge('dest') self.link_tree.unmerge('dest') self.assertFalse(os.path.exists('dest/1')) self.assertFalse(os.path.exists('dest/a/b/2')) self.assertFalse(os.path.exists('dest/a/b/3')) self.assertFalse(os.path.exists('dest/c/4')) self.assertFalse(os.path.exists('dest/c/d/5')) self.assertFalse(os.path.exists('dest/c/d/6')) self.assertFalse(os.path.exists('dest/c/d/e/7')) self.assertTrue(os.path.isdir('dest/a/b/h')) self.assertTrue(os.path.isdir('dest/f/g')) def test_ignore(self): with working_dir(self.stage.path): touchp('source/.spec') touchp('dest/.spec') self.link_tree.merge('dest', ignore=lambda x: x == '.spec') self.link_tree.unmerge('dest', ignore=lambda x: x == '.spec') self.assertFalse(os.path.exists('dest/1')) self.assertFalse(os.path.exists('dest/a')) self.assertFalse(os.path.exists('dest/c')) self.assertTrue(os.path.isfile('source/.spec')) self.assertTrue(os.path.isfile('dest/.spec'))