def expandYamlForTemplateJob(self, project, template, jobs_glob=None): dimensions = [] template_name = template['name'] # reject keys that are not useful during yaml expansion for k in ['jobs']: project.pop(k) excludes = project.pop('exclude', []) for (k, v) in project.items(): tmpk = '{{{0}}}'.format(k) if tmpk not in template_name: continue if type(v) == list: dimensions.append(zip([k] * len(v), v)) # XXX somewhat hackish to ensure we actually have a single # pass through the loop if len(dimensions) == 0: dimensions = [(("", ""),)] for values in itertools.product(*dimensions): params = copy.deepcopy(project) params = self.applyDefaults(params, template) expanded_values = {} for (k, v) in values: if isinstance(v, dict): inner_key = next(iter(v)) expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: expanded_values[k] = v params.update(expanded_values) params = deep_format(params, params) if combination_matches(params, excludes): logger.debug('Excluding combination %s', str(params)) continue allow_empty_variables = self.config \ and self.config.has_section('job_builder') \ and self.config.has_option( 'job_builder', 'allow_empty_variables') \ and self.config.getboolean( 'job_builder', 'allow_empty_variables') for key in template.keys(): if key not in params: params[key] = template[key] params['template-name'] = template_name expanded = deep_format(template, params, allow_empty_variables) job_name = expanded.get('name') if jobs_glob and not matches(job_name, jobs_glob): continue self.formatDescription(expanded) self.jobs.append(expanded)
def _expandYamlForTemplateView(self, project, template, views_glob=None): dimensions = [] template_name = template["name"] # reject keys that are not useful during yaml expansion for k in ["views"]: project.pop(k) excludes = project.pop("exclude", []) for (k, v) in project.items(): tmpk = "{{{0}}}".format(k) if tmpk not in template_name: continue if type(v) == list: dimensions.append(zip([k] * len(v), v)) # XXX somewhat hackish to ensure we actually have a single # pass through the loop if len(dimensions) == 0: dimensions = [(("", ""), )] for values in itertools.product(*dimensions): params = copy.deepcopy(project) params = self._applyDefaults(params, template) expanded_values = {} for (k, v) in values: if isinstance(v, dict): inner_key = next(iter(v)) expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: expanded_values[k] = v params.update(expanded_values) params = deep_format(params, params) if combination_matches(params, excludes): logger.debug("Excluding combination %s", str(params)) continue for key in template.keys(): if key not in params: params[key] = template[key] params["template-name"] = template_name expanded = deep_format( template, params, self.jjb_config.yamlparser["allow_empty_variables"]) view_name = expanded.get("name") if views_glob and not matches(view_name, views_glob): continue self._formatDescription(expanded) self.views.append(expanded)
def _expandYamlForTemplateJob(self, project, template, jobs_glob=None): dimensions = [] template_name = template['name'] # reject keys that are not useful during yaml expansion for k in ['jobs']: project.pop(k) excludes = project.pop('exclude', []) for (k, v) in project.items(): tmpk = '{{{0}}}'.format(k) if tmpk not in template_name: continue if type(v) == list: dimensions.append(zip([k] * len(v), v)) # XXX somewhat hackish to ensure we actually have a single # pass through the loop if len(dimensions) == 0: dimensions = [(("", ""),)] for values in itertools.product(*dimensions): params = copy.deepcopy(project) params = self._applyDefaults(params, template) try: expanded_values = {} for (k, v) in values: if isinstance(v, dict): inner_key = next(iter(v)) expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: expanded_values[k] = v except TypeError: project_name = project.pop('name') logger.error( "Exception thrown while expanding template '%s' for " "project '%s', with expansion arguments of:\n%s\n" "Original project input variables for template:\n%s\n" "Most likely the inputs have items indented incorrectly " "to describe how they should be applied.\n\nNote yaml " "'null' is mapped to python's 'None'", template_name, project_name, "".join(local_yaml.dump({k: v}, default_flow_style=False) for (k, v) in values), local_yaml.dump(project, default_flow_style=False)) raise params.update(expanded_values) try: params = deep_format(params, params) except Exception: logging.error( "Failure formatting params '%s' with itself", params) raise if combination_matches(params, excludes): logger.debug('Excluding combination %s', str(params)) continue for key in template.keys(): if key not in params: params[key] = template[key] params['template-name'] = template_name try: expanded = deep_format( template, params, self.jjb_config.yamlparser['allow_empty_variables']) except Exception: logging.error( "Failure formatting template '%s', containing '%s' with " "params '%s'", template_name, template, params) raise self._macro_registry.expand_macros(expanded, params) job_name = expanded.get('name') if jobs_glob and not matches(job_name, jobs_glob): continue self._formatDescription(expanded) self.jobs.append(expanded)
def dispatch(self, component_type, xml_parent, component, template_data={}): """This is a method that you can call from your implementation of Base.gen_xml or component. It allows modules to define a type of component, and benefit from extensibility via Python entry points and Jenkins Job Builder :ref:`Macros <macro>`. :arg string component_type: the name of the component (e.g., `builder`) :arg YAMLParser parser: the global YAML Parser :arg Element xml_parent: the parent XML element :arg dict template_data: values that should be interpolated into the component definition See :py:class:`jenkins_jobs.modules.base.Base` for how to register components of a module. See the Publishers module for a simple example of how to use this method. """ if component_type not in self.modules_by_component_type: raise JenkinsJobsException("Unknown component type: " "'{0}'.".format(component_type)) entry_point = self.modules_by_component_type[component_type] component_list_type = self.get_component_list_type(entry_point) if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) name, component_data = next(iter(component.items())) if template_data or isinstance(component_data, Jinja2Loader): # Template data contains values that should be interpolated # into the component definition. To handle Jinja2 templates # that don't contain any variables, we also deep format those. try: component_data = deep_format( component_data, template_data, self.jjb_config.yamlparser["allow_empty_variables"], ) except Exception: logging.error( "Failure formatting component ('%s') data '%s'", name, component_data, ) raise else: # The component is a simple string name, eg "run-tests" name = component component_data = {} # Look for a component function defined in an entry point eps = self._entry_points_cache.get(component_list_type) if eps is None: logging.debug("Caching entrypoints for %s" % component_list_type) module_eps = [] # auto build entry points by inferring from base component_types mod = pkg_resources.EntryPoint("__all__", entry_point.module_name, dist=entry_point.dist) Mod = mod.load() func_eps = [ Mod.__dict__.get(a) for a in dir(Mod) if isinstance(Mod.__dict__.get(a), types.FunctionType) ] for func_ep in func_eps: try: # extract entry point based on docstring name_line = func_ep.__doc__.split("\n") if not name_line[0].startswith("yaml:"): logger.debug("Ignoring '%s' as an entry point" % name_line) continue ep_name = name_line[0].split(" ")[1] except (AttributeError, IndexError): # AttributeError by docstring not being defined as # a string to have split called on it. # IndexError raised by name_line not containing anything # after the 'yaml:' string. logger.debug("Not including func '%s' as an entry point" % func_ep.__name__) continue module_eps.append( pkg_resources.EntryPoint( ep_name, entry_point.module_name, dist=entry_point.dist, attrs=(func_ep.__name__, ), )) logger.debug( "Adding auto EP '%s=%s:%s'" % (ep_name, entry_point.module_name, func_ep.__name__)) # load from explicitly defined entry points module_eps.extend( list( pkg_resources.iter_entry_points( group="jenkins_jobs.{0}".format(component_list_type)))) eps = {} for module_ep in module_eps: if module_ep.name in eps: raise JenkinsJobsException( "Duplicate entry point found for component type: " "'{0}', '{0}'," "name: '{1}'".format(component_type, name)) eps[module_ep.name] = module_ep.load() # cache both sets of entry points self._entry_points_cache[component_list_type] = eps logger.debug("Cached entry point group %s = %s", component_list_type, eps) # check for macro first component = self.parser_data.get(component_type, {}).get(name) if component: if name in eps and name not in self.masked_warned: self.masked_warned[name] = True logger.warning("You have a macro ('%s') defined for '%s' " "component type that is masking an inbuilt " "definition" % (name, component_type)) for b in component[component_list_type]: # Pass component_data in as template data to this function # so that if the macro is invoked with arguments, # the arguments are interpolated into the real defn. self.dispatch(component_type, xml_parent, b, component_data) elif name in eps: func = eps[name] func(self, xml_parent, component_data) else: raise JenkinsJobsException("Unknown entry point or macro '{0}' " "for component type: '{1}'.".format( name, component_type))
def _expandYamlForTemplateJob(self, project, template, jobs_glob=None): dimensions = [] template_name = template['name'] # reject keys that are not useful during yaml expansion for k in ['jobs']: project.pop(k) excludes = project.pop('exclude', []) for (k, v) in project.items(): tmpk = '{{{0}}}'.format(k) if tmpk not in template_name: continue if type(v) == list: dimensions.append(zip([k] * len(v), v)) # XXX somewhat hackish to ensure we actually have a single # pass through the loop if len(dimensions) == 0: dimensions = [(("", ""),)] for values in itertools.product(*dimensions): params = copy.deepcopy(project) params = self._applyDefaults(params, template) try: expanded_values = {} for (k, v) in values: if isinstance(v, dict): inner_key = next(iter(v)) expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: expanded_values[k] = v except TypeError: project_name = project.pop('name') logger.error( "Exception thrown while expanding template '%s' for " "project '%s', with expansion arguments of:\n%s\n" "Original project input variables for template:\n%s\n" "Most likely the inputs have items indented incorrectly " "to describe how they should be applied.\n\nNote yaml " "'null' is mapped to python's 'None'", template_name, project_name, "".join(local_yaml.dump({k: v}, default_flow_style=False) for (k, v) in values), local_yaml.dump(project, default_flow_style=False)) raise params.update(expanded_values) try: params = deep_format(params, params) except Exception: logging.error( "Failure formatting params '%s' with itself", params) raise if combination_matches(params, excludes): logger.debug('Excluding combination %s', str(params)) continue for key in template.keys(): if key not in params: params[key] = template[key] params['template-name'] = template_name try: expanded = deep_format( template, params, self.jjb_config.yamlparser['allow_empty_variables']) except Exception: logging.error( "Failure formatting template '%s', containing '%s' with " "params '%s'", template_name, template, params) raise job_name = expanded.get('name') if jobs_glob and not matches(job_name, jobs_glob): continue self._formatDescription(expanded) self.jobs.append(expanded)
def dispatch(self, component_type, parser, xml_parent, component, template_data={}): """This is a method that you can call from your implementation of Base.gen_xml or component. It allows modules to define a type of component, and benefit from extensibility via Python entry points and Jenkins Job Builder :ref:`Macros <macro>`. :arg string component_type: the name of the component (e.g., `builder`) :arg YAMLParser parser: the global YAML Parser :arg Element xml_parent: the parent XML element :arg dict template_data: values that should be interpolated into the component definition See :py:class:`jenkins_jobs.modules.base.Base` for how to register components of a module. See the Publishers module for a simple example of how to use this method. """ if component_type not in self.modules_by_component_type: raise JenkinsJobsException("Unknown component type: " "'{0}'.".format(component_type)) entry_point = self.modules_by_component_type[component_type] component_list_type = entry_point.load().component_list_type if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) name, component_data = next(iter(component.items())) if template_data: # Template data contains values that should be interpolated # into the component definition component_data = deep_format( component_data, template_data, self.jjb_config.yamlparser['allow_empty_variables']) else: # The component is a simple string name, eg "run-tests" name = component component_data = {} # Look for a component function defined in an entry point eps = ModuleRegistry.entry_points_cache.get(component_list_type) if eps is None: module_eps = [] # auto build entry points by inferring from base component_types mod = pkg_resources.EntryPoint( "__all__", entry_point.module_name, dist=entry_point.dist) Mod = mod.load() func_eps = [Mod.__dict__.get(a) for a in dir(Mod) if isinstance(Mod.__dict__.get(a), types.FunctionType)] for func_ep in func_eps: try: # extract entry point based on docstring name_line = func_ep.__doc__.split('\n') if not name_line[0].startswith('yaml:'): logger.debug("Ignoring '%s' as an entry point" % name_line) continue ep_name = name_line[0].split(' ')[1] except (AttributeError, IndexError): # AttributeError by docstring not being defined as # a string to have split called on it. # IndexError raised by name_line not containing anything # after the 'yaml:' string. logger.debug("Not including func '%s' as an entry point" % func_ep.__name__) continue module_eps.append( pkg_resources.EntryPoint( ep_name, entry_point.module_name, dist=entry_point.dist, attrs=(func_ep.__name__,))) logger.debug( "Adding auto EP '%s=%s:%s'" % (ep_name, entry_point.module_name, func_ep.__name__)) # load from explicitly defined entry points module_eps.extend(list(pkg_resources.iter_entry_points( group='jenkins_jobs.{0}'.format(component_list_type)))) eps = {} for module_ep in module_eps: if module_ep.name in eps: raise JenkinsJobsException( "Duplicate entry point found for component type: " "'{0}', '{0}'," "name: '{1}'".format(component_type, name)) eps[module_ep.name] = module_ep # cache both sets of entry points ModuleRegistry.entry_points_cache[component_list_type] = eps logger.debug("Cached entry point group %s = %s", component_list_type, eps) # check for macro first component = parser.data.get(component_type, {}).get(name) if component: if name in eps and name not in self.masked_warned: # Warn only once for each macro self.masked_warned[name] = True logger.warn("You have a macro ('%s') defined for '%s' " "component type that is masking an inbuilt " "definition" % (name, component_type)) for b in component[component_list_type]: # Pass component_data in as template data to this function # so that if the macro is invoked with arguments, # the arguments are interpolated into the real defn. self.dispatch(component_type, parser, xml_parent, b, component_data) elif name in eps: func = eps[name].load() func(parser, xml_parent, component_data) else: raise JenkinsJobsException("Unknown entry point or macro '{0}' " "for component type: '{1}'.". format(name, component_type))
def dispatch(self, component_type, parser, xml_parent, component, template_data={}): """This is a method that you can call from your implementation of Base.gen_xml or component. It allows modules to define a type of component, and benefit from extensibility via Python entry points and Jenkins Job Builder :ref:`Macros <macro>`. :arg string component_type: the name of the component (e.g., `builder`) :arg YAMLParser parser: the global YAML Parser :arg Element xml_parent: the parent XML element :arg dict template_data: values that should be interpolated into the component definition See :py:class:`jenkins_jobs.modules.base.Base` for how to register components of a module. See the Publishers module for a simple example of how to use this method. """ if component_type not in self.modules_by_component_type: raise JenkinsJobsException("Unknown component type: " "'{0}'.".format(component_type)) component_list_type = self.modules_by_component_type[component_type] \ .component_list_type if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) name, component_data = next(iter(component.items())) if template_data: # Template data contains values that should be interpolated # into the component definition allow_empty_variables = self.global_config \ and self.global_config.has_section('job_builder') \ and self.global_config.has_option( 'job_builder', 'allow_empty_variables') \ and self.global_config.getboolean( 'job_builder', 'allow_empty_variables') component_data = deep_format( component_data, template_data, allow_empty_variables) else: # The component is a simple string name, eg "run-tests" name = component component_data = {} # Look for a component function defined in an entry point eps = ModuleRegistry.entry_points_cache.get(component_list_type) if eps is None: module_eps = list(pkg_resources.iter_entry_points( group='jenkins_jobs.{0}'.format(component_list_type))) eps = {} for module_ep in module_eps: if module_ep.name in eps: raise JenkinsJobsException( "Duplicate entry point found for component type: " "'{0}', '{0}'," "name: '{1}'".format(component_type, name)) eps[module_ep.name] = module_ep ModuleRegistry.entry_points_cache[component_list_type] = eps logger.debug("Cached entry point group %s = %s", component_list_type, eps) if name in eps: func = eps[name].load() func(parser, xml_parent, component_data) else: # Otherwise, see if it's defined as a macro component = parser.data.get(component_type, {}).get(name) if component: for b in component[component_list_type]: # Pass component_data in as template data to this function # so that if the macro is invoked with arguments, # the arguments are interpolated into the real defn. self.dispatch(component_type, parser, xml_parent, b, component_data) else: raise JenkinsJobsException("Unknown entry point or macro '{0}'" " for component type: '{1}'.". format(name, component_type))
def _maybe_expand_macro(self, component, component_list_type, template_data=None): """For a given component, if it refers to a macro, return the components defined for that macro with template variables (if any) interpolated in. :arg str component_list_type: A string value indicating which type of component we are expanding macros for. :arg dict template_data: If component is a macro and contains template variables, use the same template data used to fill in job-template variables to fill in macro variables. """ component_copy = copy.deepcopy(component) if isinstance(component, dict): # The component is a singleton dictionary of name: # dict(args) component_name, component_data = next(iter(component_copy.items())) else: # The component is a simple string name, eg "run-tests". component_name, component_data = component_copy, None if template_data: # Address the case where a macro name contains a variable to be # interpolated by template variables. component_name = deep_format(component_name, template_data, True) # Check that the component under consideration actually is a # macro. if not self._is_macro(component_name, component_list_type): return None # Warn if the macro shadows an actual module type name for this # component list type. if ModuleRegistry.is_module_name(component_name, component_list_type): self._mask_warned[component_name] = True logger.warning( "You have a macro ('%s') defined for '%s' " "component list type that is masking an inbuilt " "definition" % (component_name, component_list_type)) macro_component_list = self._get_macro_components(component_name, component_list_type) # If macro instance contains component_data, interpolate that # into macro components. if component_data: # Also use template_data, but prefer data obtained directly from # the macro instance. if template_data: template_data = copy.deepcopy(template_data) template_data.update(component_data) macro_component_list = deep_format( macro_component_list, template_data, False) else: macro_component_list = deep_format( macro_component_list, component_data, False) return macro_component_list
def dispatch(self, component_type, parser, xml_parent, component, template_data={}): """This is a method that you can call from your implementation of Base.gen_xml or component. It allows modules to define a type of component, and benefit from extensibility via Python entry points and Jenkins Job Builder :ref:`Macros <macro>`. :arg string component_type: the name of the component (e.g., `builder`) :arg YAMLParser parser: the global YAML Parser :arg Element xml_parent: the parent XML element :arg dict template_data: values that should be interpolated into the component definition See :py:class:`jenkins_jobs.modules.base.Base` for how to register components of a module. See the Publishers module for a simple example of how to use this method. """ if component_type not in self.modules_by_component_type: raise JenkinsJobsException("Unknown component type: " "'{0}'.".format(component_type)) component_list_type = self.modules_by_component_type[component_type] \ .component_list_type if isinstance(component, dict): # The component is a singleton dictionary of name: dict(args) name, component_data = next(iter(component.items())) if template_data: # Template data contains values that should be interpolated # into the component definition allow_empty_variables = self.global_config \ and self.global_config.has_section('job_builder') \ and self.global_config.has_option( 'job_builder', 'allow_empty_variables') \ and self.global_config.getboolean( 'job_builder', 'allow_empty_variables') component_data = deep_format(component_data, template_data, allow_empty_variables) else: # The component is a simple string name, eg "run-tests" name = component component_data = {} # Look for a component function defined in an entry point eps = ModuleRegistry.entry_points_cache.get(component_list_type) if eps is None: module_eps = list( pkg_resources.iter_entry_points( group='jenkins_jobs.{0}'.format(component_list_type))) eps = {} for module_ep in module_eps: if module_ep.name in eps: raise JenkinsJobsException( "Duplicate entry point found for component type: " "'{0}', '{0}'," "name: '{1}'".format(component_type, name)) eps[module_ep.name] = module_ep ModuleRegistry.entry_points_cache[component_list_type] = eps logger.debug("Cached entry point group %s = %s", component_list_type, eps) if name in eps: func = eps[name].load() func(parser, xml_parent, component_data) else: # Otherwise, see if it's defined as a macro component = parser.data.get(component_type, {}).get(name) if component: for b in component[component_list_type]: # Pass component_data in as template data to this function # so that if the macro is invoked with arguments, # the arguments are interpolated into the real defn. self.dispatch(component_type, parser, xml_parent, b, component_data) else: raise JenkinsJobsException( "Unknown entry point or macro '{0}'" " for component type: '{1}'.".format(name, component_type))
def _maybe_expand_macro(self, component, component_list_type, template_data=None): """For a given component, if it refers to a macro, return the components defined for that macro with template variables (if any) interpolated in. :arg str component_list_type: A string value indicating which type of component we are expanding macros for. :arg dict template_data: If component is a macro and contains template variables, use the same template data used to fill in job-template variables to fill in macro variables. """ component_copy = copy.deepcopy(component) if isinstance(component, dict): # The component is a singleton dictionary of name: # dict(args) component_name, component_data = next(iter(component_copy.items())) else: # The component is a simple string name, eg "run-tests". component_name, component_data = component_copy, None if template_data: # Address the case where a macro name contains a variable to be # interpolated by template variables. component_name = deep_format(component_name, template_data, True) # Check that the component under consideration actually is a # macro. if not self._is_macro(component_name, component_list_type): return None # Warn if the macro shadows an actual module type name for this # component list type. if ModuleRegistry.is_module_name(component_name, component_list_type): self._mask_warned[component_name] = True logger.warning("You have a macro ('%s') defined for '%s' " "component list type that is masking an inbuilt " "definition" % (component_name, component_list_type)) macro_component_list = self._get_macro_components( component_name, component_list_type) # If macro instance contains component_data, interpolate that # into macro components. if component_data: # Also use template_data, but prefer data obtained directly from # the macro instance. if template_data: template_data = copy.deepcopy(template_data) template_data.update(component_data) macro_component_list = deep_format(macro_component_list, template_data, False) else: macro_component_list = deep_format(macro_component_list, component_data, False) return macro_component_list