def update_index(args): """Update the index of an s3 buildcache""" s3, bucket_name = get_s3_session(args.endpoint_url) bucket = s3.Bucket(bucket_name) exists = True try: s3.meta.client.head_bucket(Bucket=bucket_name) except botocore.exceptions.ClientError as e: # If a client error is thrown, then check that it was a 404 error. # If it was a 404 error, then the bucket does not exist. error_code = e.response['Error']['Code'] if error_code == '404': exists = False if not exists: tty.error('S3 bucket "{0}" does not exist'.format(bucket_name)) sys.exit(1) build_cache_dir = os.path.join('mirror', bindist.build_cache_relative_path()) spec_yaml_regex = re.compile( '{0}/(.+\\.spec\\.yaml)$'.format(build_cache_dir)) spack_regex = re.compile('{0}/([^/]+)/.+\\.spack$'.format(build_cache_dir)) top_level_keys = set() for key in bucket.objects.all(): m = spec_yaml_regex.search(key.key) if m: top_level_keys.add(m.group(1)) print(m.group(1)) continue m = spack_regex.search(key.key) if m: top_level_keys.add(m.group(1)) print(m.group(1)) continue index_data = { 'top_level_keys': top_level_keys, } env = template_engine.make_environment() template_dir = 'misc' index_template = os.path.join(template_dir, 'buildcache_index.html') t = env.get_template(index_template) contents = t.render(index_data) index_key = os.path.join(build_cache_dir, 'index.html') tty.debug('Generated index:') tty.debug(contents) tty.debug('Pushing it to {0} -> {1}'.format(bucket_name, index_key)) s3_obj = s3.Object(bucket_name, index_key) s3_obj.put(Body=contents, ACL='public-read')
def bootstrap(self): """Information related to the build image.""" images_config = self.container_config['images'] bootstrap_recipe = None if self.bootstrap_image: config_args = _spack_checkout_config(images_config) command = checkout_command(*config_args) template_path = bootstrap_template_for(self.operating_system_key) env = tengine.make_environment() context = {"bootstrap": { "image": self.bootstrap_image, "spack_checkout": command }} bootstrap_recipe = env.get_template(template_path).render(**context) Bootstrap = collections.namedtuple('Bootstrap', ['image', 'recipe']) return Bootstrap(image=self.bootstrap_image, recipe=bootstrap_recipe)
def test_template_retrieval(self): """Tests the template retrieval mechanism hooked into config files""" # Check the directories are correct template_dirs = spack.config.get('config:template_dirs') template_dirs = [canonicalize_path(x) for x in template_dirs] assert len(template_dirs) == 3 env = tengine.make_environment(template_dirs) # Retrieve a.txt, which resides in the second # template directory specified in the mock configuration template = env.get_template('a.txt') text = template.render({'word': 'world'}) assert 'Hello world!' == text # Retrieve b.txt, which resides in the third # template directory specified in the mock configuration template = env.get_template('b.txt') text = template.render({'word': 'world'}) assert 'Howdy world!' == text
def write(self, overwrite=False): """Writes the module file. Args: overwrite (bool): if True it is fine to overwrite an already existing file. If False the operation is skipped an we print a warning to the user. """ # Return immediately if the module is blacklisted if self.conf.blacklisted: msg = '\tNOT WRITING: {0} [BLACKLISTED]' tty.debug(msg.format(self.spec.cshort_spec)) return # Print a warning in case I am accidentally overwriting # a module file that is already there (name clash) if not overwrite and os.path.exists(self.layout.filename): message = 'Module file already exists : skipping creation\n' message += 'file : {0.filename}\n' message += 'spec : {0.spec}' tty.warn(message.format(self.layout)) return # If we are here it means it's ok to write the module file msg = '\tWRITE: {0} [{1}]' tty.debug(msg.format(self.spec.cshort_spec, self.layout.filename)) # If the directory where the module should reside does not exist # create it module_dir = os.path.dirname(self.layout.filename) if not os.path.exists(module_dir): llnl.util.filesystem.mkdirp(module_dir) # Get the template for the module template_name = self._get_template() import jinja2 try: env = tengine.make_environment() template = env.get_template(template_name) except jinja2.TemplateNotFound: # If the template was not found raise an exception with a little # more information msg = 'template \'{0}\' was not found for \'{1}\'' name = type(self).__name__ msg = msg.format(template_name, name) raise ModulesTemplateNotFoundError(msg) # Construct the context following the usual hierarchy of updates: # 1. start with the default context from the module writer class # 2. update with package specific context # 3. update with 'modules.yaml' specific context context = self.context.to_dict() # Attribute from package module_name = str(self.module.__name__).split('.')[-1] attr_name = '{0}_context'.format(module_name) pkg_update = getattr(self.spec.package, attr_name, {}) context.update(pkg_update) # Context key in modules.yaml conf_update = self.conf.context context.update(conf_update) # Render the template text = template.render(context) # Write it to file with open(self.layout.filename, 'w') as f: f.write(text) # Set the file permissions of the module to match that of the package if os.path.exists(self.layout.filename): fp.set_permissions_by_spec(self.layout.filename, self.spec)
def write(self, overwrite=False): """Writes the module file. Args: overwrite (bool): if True it is fine to overwrite an already existing file. If False the operation is skipped an we print a warning to the user. """ # Return immediately if the module is blacklisted if self.conf.blacklisted: msg = '\tNOT WRITING: {0} [BLACKLISTED]' tty.debug(msg.format(self.spec.cshort_spec)) return # Print a warning in case I am accidentally overwriting # a module file that is already there (name clash) if not overwrite and os.path.exists(self.layout.filename): message = 'Module file already exists : skipping creation\n' message += 'file : {0.filename}\n' message += 'spec : {0.spec}' tty.warn(message.format(self.layout)) return # If we are here it means it's ok to write the module file msg = '\tWRITE: {0} [{1}]' tty.debug(msg.format(self.spec.cshort_spec, self.layout.filename)) # If the directory where the module should reside does not exist # create it module_dir = os.path.dirname(self.layout.filename) if not os.path.exists(module_dir): llnl.util.filesystem.mkdirp(module_dir) # Get the template for the module template_name = self._get_template() try: env = tengine.make_environment() template = env.get_template(template_name) except tengine.TemplateNotFound: # If the template was not found raise an exception with a little # more information msg = 'template \'{0}\' was not found for \'{1}\'' name = type(self).__name__ msg = msg.format(template_name, name) raise ModulesTemplateNotFoundError(msg) # Construct the context following the usual hierarchy of updates: # 1. start with the default context from the module writer class # 2. update with package specific context # 3. update with 'modules.yaml' specific context context = self.context.to_dict() # Attribute from package module_name = str(self.module.__name__).split('.')[-1] attr_name = '{0}_context'.format(module_name) pkg_update = getattr(self.spec.package, attr_name, {}) context.update(pkg_update) # Context key in modules.yaml conf_update = self.conf.context context.update(conf_update) # Render the template text = template.render(context) # Write it to file with open(self.layout.filename, 'w') as f: f.write(text)
def __call__(self): """Returns the recipe as a string""" env = tengine.make_environment() t = env.get_template(self.template_name) return t.render(**self.to_dict())