def buildcache_update_index(args): """Update a buildcache index.""" outdir = '.' if args.mirror_url: outdir = args.mirror_url mirror = spack.mirror.MirrorCollection().lookup(outdir) outdir = url_util.format(mirror.push_url) bindist.generate_package_index( url_util.join(outdir, bindist.build_cache_relative_path()))
def update_index(mirror_url, update_keys=False): mirror = spack.mirror.MirrorCollection().lookup(mirror_url) outdir = url_util.format(mirror.push_url) bindist.generate_package_index( url_util.join(outdir, bindist.build_cache_relative_path())) if update_keys: keys_url = url_util.join(outdir, bindist.build_cache_relative_path(), bindist.build_cache_keys_relative_path()) bindist.generate_key_index(keys_url)
def test_generate_indices_key_error(monkeypatch, capfd): def mock_list_url(url, recursive=False): print('mocked list_url({0}, {1})'.format(url, recursive)) raise KeyError('Test KeyError handling') monkeypatch.setattr(web_util, 'list_url', mock_list_url) test_url = 'file:///fake/keys/dir' # Make sure generate_key_index handles the KeyError bindist.generate_key_index(test_url) err = capfd.readouterr()[1] assert 'Warning: No keys at {0}'.format(test_url) in err # Make sure generate_package_index handles the KeyError bindist.generate_package_index(test_url) err = capfd.readouterr()[1] assert 'Warning: No packages at {0}'.format(test_url) in err
def test_generate_indices_exception(monkeypatch, capfd): def mock_list_url(url, recursive=False): print('mocked list_url({0}, {1})'.format(url, recursive)) raise Exception('Test Exception handling') monkeypatch.setattr(web_util, 'list_url', mock_list_url) test_url = 'file:///fake/keys/dir' # Make sure generate_key_index handles the Exception bindist.generate_key_index(test_url) err = capfd.readouterr()[1] expect = 'Encountered problem listing keys at {0}'.format(test_url) assert expect in err # Make sure generate_package_index handles the Exception bindist.generate_package_index(test_url) err = capfd.readouterr()[1] expect = 'Encountered problem listing packages at {0}'.format(test_url) assert expect in err
def createtarball(args): """create a binary package from an existing install""" if args.spec_yaml: packages = set() tty.msg('createtarball, reading spec from {0}'.format(args.spec_yaml)) with open(args.spec_yaml, 'r') as fd: yaml_text = fd.read() tty.debug('createtarball read spec yaml:') tty.debug(yaml_text) s = Spec.from_yaml(yaml_text) packages.add('/{0}'.format(s.dag_hash())) elif args.packages: packages = args.packages else: tty.die("build cache file creation requires at least one" + " installed package argument or else path to a" + " yaml file containing a spec to install") pkgs = set(packages) specs = set() outdir = '.' if args.directory: outdir = args.directory mirror = spack.mirror.MirrorCollection().lookup(outdir) outdir = url_util.format(mirror.push_url) signkey = None if args.key: signkey = args.key # restrict matching to current environment if one is active env = ev.get_env(args, 'buildcache create') matches = find_matching_specs(pkgs, env=env) if matches: tty.debug('Found at least one matching spec') for match in matches: tty.debug('examining match {0}'.format(match.format())) if match.external or match.virtual: tty.debug('skipping external or virtual spec %s' % match.format()) else: tty.debug('adding matching spec %s' % match.format()) if "package" in args.target_type: specs.add(match) if "dependencies" not in args.target_type: # if the user does not want dependencies, stop here continue tty.debug('recursing dependencies') for d, node in match.traverse(order='post', depth=True, deptype=('link', 'run')): if node.external or node.virtual: tty.debug('skipping external or virtual dependency %s' % node.format()) else: tty.debug('adding dependency %s' % node.format()) specs.add(node) tty.debug('writing tarballs to %s/build_cache' % outdir) f_create = ft.partial(create_single_tarball, outdir=outdir, force=args.force, relative=args.rel, unsigned=args.unsigned, allow_root=args.allow_root, signkey=signkey, rebuild_index=args.rebuild_index and args.jobs == 1, catch_exceptions=args.jobs != 1) # default behavior (early termination) for one job if args.jobs == 1: for spec in specs: f_create(spec) else: # currently, specs cause an infinite recursion bug when pickled # -> as multiprocessing uses pickle internally, we need to transform # specs prior to distributing the work via worker pool # TODO: check if specs can be pickled specs = [s.to_dict() for s in specs] pool = NoDaemonPool(args.jobs if args.jobs > 1 else mp.cpu_count()) # chunksize=1 because we do not want to pre-allocate specs to workers # (since each package will need a different amount of time to be # compressed) retvals = pool.map(f_create, specs, chunksize=1) errors = [rv["error"] for rv in retvals if rv["error"] is not None] list(map(tty.error, errors)) if len(errors) > 0: sys.exit(1) # perform rebuild of index unless user requested not to if args.rebuild_index: bindist.generate_package_index(outdir)