def update_index(args):
"""Update the index of an s3 buildcache"""
s3, bucket_name = get_s3_session(args.endpoint_url)
bucket = s3.Bucket(bucket_name)
exists = True
try:
s3.meta.client.head_bucket(Bucket=bucket_name)
except botocore.exceptions.ClientError as e:
# If a client error is thrown, then check that it was a 404 error.
# If it was a 404 error, then the bucket does not exist.
error_code = e.response['Error']['Code']
if error_code == '404':
exists = False
if not exists:
tty.error('S3 bucket "{0}" does not exist'.format(bucket_name))
sys.exit(1)
build_cache_dir = os.path.join('mirror',
bindist.build_cache_relative_path())
spec_yaml_regex = re.compile(
'{0}/(.+\\.spec\\.yaml)$'.format(build_cache_dir))
spack_regex = re.compile('{0}/([^/]+)/.+\\.spack$'.format(build_cache_dir))
top_level_keys = set()
for key in bucket.objects.all():
m = spec_yaml_regex.search(key.key)
if m:
top_level_keys.add(m.group(1))
print(m.group(1))
continue
m = spack_regex.search(key.key)
if m:
top_level_keys.add(m.group(1))
print(m.group(1))
continue
index_data = {
'top_level_keys': top_level_keys,
}
env = template_engine.make_environment()
template_dir = 'misc'
index_template = os.path.join(template_dir, 'buildcache_index.html')
t = env.get_template(index_template)
contents = t.render(index_data)
index_key = os.path.join(build_cache_dir, 'index.html')
tty.debug('Generated index:')
tty.debug(contents)
tty.debug('Pushing it to {0} -> {1}'.format(bucket_name, index_key))
s3_obj = s3.Object(bucket_name, index_key)
s3_obj.put(Body=contents, ACL='public-read')
def bootstrap(self):
"""Information related to the build image."""
images_config = self.container_config['images']
bootstrap_recipe = None
if self.bootstrap_image:
config_args = _spack_checkout_config(images_config)
command = checkout_command(*config_args)
template_path = bootstrap_template_for(self.operating_system_key)
env = tengine.make_environment()
context = {"bootstrap": {
"image": self.bootstrap_image,
"spack_checkout": command
}}
bootstrap_recipe = env.get_template(template_path).render(**context)
Bootstrap = collections.namedtuple('Bootstrap', ['image', 'recipe'])
return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe)
def test_template_retrieval(self):
"""Tests the template retrieval mechanism hooked into config files"""
# Check the directories are correct
template_dirs = spack.config.get('config:template_dirs')
template_dirs = [canonicalize_path(x) for x in template_dirs]
assert len(template_dirs) == 3
env = tengine.make_environment(template_dirs)
# Retrieve a.txt, which resides in the second
# template directory specified in the mock configuration
template = env.get_template('a.txt')
text = template.render({'word': 'world'})
assert 'Hello world!' == text
# Retrieve b.txt, which resides in the third
# template directory specified in the mock configuration
template = env.get_template('b.txt')
text = template.render({'word': 'world'})
assert 'Howdy world!' == text
def test_template_retrieval(self):
"""Tests the template retrieval mechanism hooked into config files"""
# Check the directories are correct
template_dirs = spack.config.get('config:template_dirs')
template_dirs = [canonicalize_path(x) for x in template_dirs]
assert len(template_dirs) == 3
env = tengine.make_environment(template_dirs)
# Retrieve a.txt, which resides in the second
# template directory specified in the mock configuration
template = env.get_template('a.txt')
text = template.render({'word': 'world'})
assert 'Hello world!' == text
# Retrieve b.txt, which resides in the third
# template directory specified in the mock configuration
template = env.get_template('b.txt')
text = template.render({'word': 'world'})
assert 'Howdy world!' == text
def write(self, overwrite=False):
"""Writes the module file.
Args:
overwrite (bool): if True it is fine to overwrite an already
existing file. If False the operation is skipped an we print
a warning to the user.
"""
# Return immediately if the module is blacklisted
if self.conf.blacklisted:
msg = '\tNOT WRITING: {0} [BLACKLISTED]'
tty.debug(msg.format(self.spec.cshort_spec))
return
# Print a warning in case I am accidentally overwriting
# a module file that is already there (name clash)
if not overwrite and os.path.exists(self.layout.filename):
message = 'Module file already exists : skipping creation\n'
message += 'file : {0.filename}\n'
message += 'spec : {0.spec}'
tty.warn(message.format(self.layout))
return
# If we are here it means it's ok to write the module file
msg = '\tWRITE: {0} [{1}]'
tty.debug(msg.format(self.spec.cshort_spec, self.layout.filename))
# If the directory where the module should reside does not exist
# create it
module_dir = os.path.dirname(self.layout.filename)
if not os.path.exists(module_dir):
llnl.util.filesystem.mkdirp(module_dir)
# Get the template for the module
template_name = self._get_template()
import jinja2
try:
env = tengine.make_environment()
template = env.get_template(template_name)
except jinja2.TemplateNotFound:
# If the template was not found raise an exception with a little
# more information
msg = 'template \'{0}\' was not found for \'{1}\''
name = type(self).__name__
msg = msg.format(template_name, name)
raise ModulesTemplateNotFoundError(msg)
# Construct the context following the usual hierarchy of updates:
# 1. start with the default context from the module writer class
# 2. update with package specific context
# 3. update with 'modules.yaml' specific context
context = self.context.to_dict()
# Attribute from package
module_name = str(self.module.__name__).split('.')[-1]
attr_name = '{0}_context'.format(module_name)
pkg_update = getattr(self.spec.package, attr_name, {})
context.update(pkg_update)
# Context key in modules.yaml
conf_update = self.conf.context
context.update(conf_update)
# Render the template
text = template.render(context)
# Write it to file
with open(self.layout.filename, 'w') as f:
f.write(text)
# Set the file permissions of the module to match that of the package
if os.path.exists(self.layout.filename):
fp.set_permissions_by_spec(self.layout.filename, self.spec)
def write(self, overwrite=False):
"""Writes the module file.
Args:
overwrite (bool): if True it is fine to overwrite an already
existing file. If False the operation is skipped an we print
a warning to the user.
"""
# Return immediately if the module is blacklisted
if self.conf.blacklisted:
msg = '\tNOT WRITING: {0} [BLACKLISTED]'
tty.debug(msg.format(self.spec.cshort_spec))
return
# Print a warning in case I am accidentally overwriting
# a module file that is already there (name clash)
if not overwrite and os.path.exists(self.layout.filename):
message = 'Module file already exists : skipping creation\n'
message += 'file : {0.filename}\n'
message += 'spec : {0.spec}'
tty.warn(message.format(self.layout))
return
# If we are here it means it's ok to write the module file
msg = '\tWRITE: {0} [{1}]'
tty.debug(msg.format(self.spec.cshort_spec, self.layout.filename))
# If the directory where the module should reside does not exist
# create it
module_dir = os.path.dirname(self.layout.filename)
if not os.path.exists(module_dir):
llnl.util.filesystem.mkdirp(module_dir)
# Get the template for the module
template_name = self._get_template()
try:
env = tengine.make_environment()
template = env.get_template(template_name)
except tengine.TemplateNotFound:
# If the template was not found raise an exception with a little
# more information
msg = 'template \'{0}\' was not found for \'{1}\''
name = type(self).__name__
msg = msg.format(template_name, name)
raise ModulesTemplateNotFoundError(msg)
# Construct the context following the usual hierarchy of updates:
# 1. start with the default context from the module writer class
# 2. update with package specific context
# 3. update with 'modules.yaml' specific context
context = self.context.to_dict()
# Attribute from package
module_name = str(self.module.__name__).split('.')[-1]
attr_name = '{0}_context'.format(module_name)
pkg_update = getattr(self.spec.package, attr_name, {})
context.update(pkg_update)
# Context key in modules.yaml
conf_update = self.conf.context
context.update(conf_update)
# Render the template
text = template.render(context)
# Write it to file
with open(self.layout.filename, 'w') as f:
f.write(text)
def __call__(self):
"""Returns the recipe as a string"""
env = tengine.make_environment()
t = env.get_template(self.template_name)
return t.render(**self.to_dict())
