class ReleaseTraitTransformer(TraitTransformer): name = 'release' def inject_steps(self): # inject 'release' step self.release_step = PipelineStep( name='release', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) yield self.release_step def process_pipeline_args(self, pipeline_args: 'JobVariant'): # we depend on all other steps for step in pipeline_args.steps(): self.release_step._add_dependency(step) # a 'release job' should only be triggered automatically if explicitly configured main_repo = pipeline_args.main_repository() if main_repo: if 'trigger' not in pipeline_args.raw['repo']: main_repo._trigger = False @classmethod def dependencies(cls): return {'version'} @classmethod def order_dependencies(cls): return {'publish'}
class SourceScanTraitTransformer(TraitTransformer): name = 'scan_sources' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) def inject_steps(self): self.source_scan_step = PipelineStep( name='scan_sources', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.source_scan_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.source_scan_step.set_timeout(duration_string='18h') yield self.source_scan_step def process_pipeline_args(self, pipeline_args: JobVariant): # our step depends on dependency descriptor step component_descriptor_step = pipeline_args.step('component_descriptor') self.source_scan_step._add_dependency(component_descriptor_step) @classmethod def dependencies(cls): return {'component_descriptor'}
class ImageScanTraitTransformer(TraitTransformer): name = 'image_scan' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) def inject_steps(self): self.image_scan_step = PipelineStep( name='scan_container_images', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.image_scan_step.add_input(*COMPONENT_DESCRIPTOR_DIR_INPUT) self.image_scan_step.set_timeout(duration_string='12h') yield self.image_scan_step def process_pipeline_args(self, pipeline_args: 'JobVariant'): # our step depends on dependency descriptor step component_descriptor_step = pipeline_args.step('component_descriptor') self.image_scan_step._add_dependency(component_descriptor_step) @classmethod def dependencies(cls): return {'component_descriptor'}
class DraftReleaseTraitTransformer(TraitTransformer): name = 'draft_release' def inject_steps(self): # inject 'release' step self.release_step = PipelineStep( name='create_draft_release_notes', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, pull_request_notification_policy=PullRequestNotificationPolicy.NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, ) self.release_step.set_timeout(duration_string='10m') yield self.release_step def process_pipeline_args(self, pipeline_args: JobVariant): cd_trait = pipeline_args.trait('component_descriptor') cd_step = pipeline_args.step(cd_trait.step_name()) self.release_step._add_dependency(cd_step) @classmethod def dependencies(cls): return {'version', 'component_descriptor'}
def inject_steps(self): # 'publish' step publish_step = PipelineStep( name='publish', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS, script_type=ScriptType.BOURNE_SHELL, ) publish_step.set_timeout(duration_string='4h') # 'prepare' step prepare_step = PipelineStep( name='prepare', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.BOURNE_SHELL, ) prepare_step.set_timeout(duration_string='30m') publish_step._add_dependency(prepare_step) yield prepare_step yield publish_step
class ImageScanTraitTransformer(TraitTransformer): name = 'image_scan' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) def inject_steps(self): self.image_scan_step = PipelineStep( name='scan_container_images', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.image_scan_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.image_scan_step.set_timeout(duration_string='12h') yield self.image_scan_step def process_pipeline_args(self, pipeline_args: JobVariant): # our step depends on dependency descriptor step component_descriptor_step = pipeline_args.step('component_descriptor') self.image_scan_step._add_dependency(component_descriptor_step) for trait_name in self.trait.trait_depends(): if not pipeline_args.has_trait(trait_name): raise ModelValidationError( f'dependency towards absent trait: {trait_name}') depended_on_trait = pipeline_args.trait(trait_name) # XXX refactor Trait/TraitTransformer transformer = depended_on_trait.transformer() # XXX step-injection may have (unintended) side-effects :-/ depended_on_step_names = { step.name for step in transformer.inject_steps() } for step in pipeline_args.steps(): if not step.name in depended_on_step_names: continue self.image_scan_step._add_dependency(step) # prevent cyclic dependencies (from auto-injected depends) if self.image_scan_step.name in step.depends(): step._remove_dependency(self.image_scan_step) @classmethod def dependencies(cls): return {'component_descriptor'} @classmethod def order_dependencies(cls): # required in case image-scanning should be done after publish # (-> auto-injected dependency for "prepare"-step towards _all_ steps) return {'publish'}
class UpdateComponentDependenciesTraitTransformer(TraitTransformer): name = 'update_component_deps' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) @classmethod def order_dependencies(cls): return {'component_descriptor'} @classmethod def dependencies(cls): return {'component_descriptor'} def inject_steps(self): if self.trait.set_dependency_version_script_container_image(): privilege_mode = PrivilegeMode.PRIVILEGED else: privilege_mode = PrivilegeMode.UNPRIVILEGED # declare no dependencies --> run asap, but do not block other steps self.update_component_deps_step = PipelineStep( name='update_component_dependencies', raw_dict={ 'privilege_mode': privilege_mode, }, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3) self.update_component_deps_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.update_component_deps_step.set_timeout(duration_string='30m') for name, value in self.trait.vars().items(): self.update_component_deps_step.variables()[name] = value yield self.update_component_deps_step def process_pipeline_args(self, pipeline_args: JobVariant): # our step depends on dependendency descriptor step component_descriptor_step = pipeline_args.step( concourse.model.traits.component_descriptor. DEFAULT_COMPONENT_DESCRIPTOR_STEP_NAME) self.update_component_deps_step._add_dependency( component_descriptor_step) upstream_component_name = self.trait.upstream_component_name() if upstream_component_name: self.update_component_deps_step.variables( )['UPSTREAM_COMPONENT_NAME'] = '"{cn}"'.format( cn=upstream_component_name, )
def inject_steps(self): # 'publish' step publish_step = PipelineStep( name='publish', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS, injected_by_trait=self.name, script_type=ScriptType.BOURNE_SHELL, ) publish_step.set_timeout(duration_string='4h') # 'prepare' step prepare_step = PipelineStep( name='prepare', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, injected_by_trait=self.name, script_type=ScriptType.BOURNE_SHELL, ) prepare_step.set_timeout(duration_string='30m') publish_step._add_dependency(prepare_step) if self.trait.oci_builder() is OciBuilder.KANIKO: with open(concourse.paths.last_released_tag_file) as f: last_tag = f.read().strip() kaniko_image_ref = f'eu.gcr.io/gardener-project/cc/job-image-kaniko:{last_tag}' for img in self.trait.dockerimages(): build_step = PipelineStep( name=f'build_oci_image_{img.name()}', raw_dict={ 'image': kaniko_image_ref, }, is_synthetic=True, notification_policy=StepNotificationPolicy. NOTIFY_PULL_REQUESTS, injected_by_trait=self.name, script_type=ScriptType.PYTHON3, extra_args={ 'image_descriptor': img, }) build_step._add_dependency(prepare_step) self._build_steps.append(build_step) yield build_step yield prepare_step if self.trait.oci_builder() is OciBuilder.CONCOURSE_IMAGE_RESOURCE: yield publish_step
class ReleaseTraitTransformer(TraitTransformer): name = 'release' def __init__(self, trait: ReleaseTrait, *args, **kwargs): super().__init__(*args, **kwargs) self.trait = trait def inject_steps(self): if self.trait.release_callback_image_reference(): # we need privileged container in order to run callback in container privilege_mode = PrivilegeMode.PRIVILEGED else: privilege_mode = PrivilegeMode.UNPRIVILEGED # inject 'release' step self.release_step = PipelineStep( name='release', raw_dict={ 'privilege_mode': privilege_mode, }, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, ) yield self.release_step def process_pipeline_args(self, pipeline_args: JobVariant): # we depend on all other steps for step in pipeline_args.steps(): self.release_step._add_dependency(step) # a 'release job' should only be triggered automatically if explicitly configured main_repo = pipeline_args.main_repository() if main_repo: if 'trigger' not in pipeline_args.raw['repo']: main_repo._trigger = False @classmethod def dependencies(cls): return {'version', 'component_descriptor'} @classmethod def order_dependencies(cls): return {'publish'}
class UpdateComponentDependenciesTraitTransformer(TraitTransformer): name = 'update_component_deps' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) @classmethod def order_dependencies(cls): return {'component_descriptor'} @classmethod def dependencies(cls): return {'component_descriptor'} def inject_steps(self): # declare no dependencies --> run asap, but do not block other steps self.update_component_deps_step = PipelineStep( name='update_component_dependencies', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3 ) self.update_component_deps_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor.ENV_VAR_NAME, ) self.update_component_deps_step.set_timeout(duration_string='30m') for name, value in self.trait.vars().items(): self.update_component_deps_step.variables()[name] = value yield self.update_component_deps_step def process_pipeline_args(self, pipeline_args: JobVariant): # our step depends on dependendency descriptor step component_descriptor_step = pipeline_args.step('component_descriptor') self.update_component_deps_step._add_dependency(component_descriptor_step) upstream_component_name = self.trait.upstream_component_name() if upstream_component_name: self.update_component_deps_step.variables()['UPSTREAM_COMPONENT_NAME'] = '"{cn}"'.format( cn=upstream_component_name, )
class ComponentDescriptorTraitTransformer(TraitTransformer): name = 'component_descriptor' def __init__(self, trait: ComponentDescriptorTrait, *args, **kwargs): super().__init__(*args, **kwargs) self.trait = not_none(trait) def inject_steps(self): self.descriptor_step = PipelineStep( name=self.trait.step_name(), raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, injected_by_trait=self.name, script_type=ScriptType.PYTHON3, ) self.descriptor_step.add_output( name=DIR_NAME, variable_name=ENV_VAR_NAME, ) self.descriptor_step.set_timeout(duration_string='45m') yield self.descriptor_step def process_pipeline_args(self, pipeline_args: 'JobVariant'): if pipeline_args.has_step('release'): release_step = pipeline_args.step('release') release_step.add_input( name=DIR_NAME, variable_name=ENV_VAR_NAME, ) if pipeline_args.has_trait('draft_release'): draft_release_step = pipeline_args.step( 'create_draft_release_notes') draft_release_step.add_input( name=DIR_NAME, variable_name=ENV_VAR_NAME, ) # inject component_name if not configured if not self.trait.raw.get('component_name'): main_repo = pipeline_args.main_repository() component_name = '/'.join(( main_repo.repo_hostname(), main_repo.repo_path(), )) self.trait.raw['component_name'] = component_name # add configured (step-)inputs for step_input in self.trait.inputs(): if not step_input.type == 'step': raise NotImplementedError(step_input.type) try: step: PipelineStep = pipeline_args.step(step_input.step_name) except KeyError as ke: raise ValueError( f'no such step: {step_input.step_name=}') from ke self.descriptor_step._add_dependency(step) if step_input.output_name: output_name = step_input.output_name else: # choose only output if omitted outputs = { name: v for name, v in step.outputs().items() if not name == 'on_error_dir' # XXX hack hack hack } if len(outputs) < 1: raise ValueError(f'{step.name=} does not have any outputs') elif len(outputs) > 1: raise ValueError( f'{step.name=} has more than one output (need to tell step_name)' ) output_name = next(outputs.keys().__iter__()) self.descriptor_step.add_input( name=output_name, variable_name=output_name, ) @classmethod def dependencies(cls): return {'version'} @classmethod def order_dependencies(cls): # dependency is required, as we need to patch the 'release' step return {'release'}
def inject_steps(self): publish_step = PipelineStep( name='publish', raw_dict={}, is_synthetic=True, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, extra_args={ 'publish_trait': self.trait, }, ) publish_step.set_timeout(duration_string='4h') self._publish_step = publish_step # 'prepare' step prepare_step = PipelineStep( name='prepare', raw_dict={}, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.BOURNE_SHELL, ) prepare_step.set_timeout(duration_string='30m') publish_step._add_dependency(prepare_step) if (oci_builder := self.trait.oci_builder()) in ( OciBuilder.KANIKO, OciBuilder.DOCKER, OciBuilder.DOCKER_BUILDX, ): if oci_builder is OciBuilder.KANIKO: with open(concourse.paths.last_released_tag_file) as f: last_tag = f.read().strip() kaniko_image_ref = f'eu.gcr.io/gardener-project/cc/job-image-kaniko:{last_tag}' elif oci_builder in (OciBuilder.DOCKER, OciBuilder.DOCKER_BUILDX): kaniko_image_ref = None else: raise NotImplementedError(oci_builder) for img in self.trait.dockerimages(): build_step = PipelineStep( name=f'build_oci_image_{img.name()}', raw_dict={ 'image': kaniko_image_ref, 'privilege_mode': PrivilegeMode.PRIVILEGED if oci_builder in (OciBuilder.DOCKER, OciBuilder.DOCKER_BUILDX) else PrivilegeMode.UNPRIVILEGED, }, is_synthetic=True, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, extra_args={ 'image_descriptor': img, }) build_step._add_dependency(prepare_step) self._build_steps.append(build_step) yield build_step publish_step._add_dependency(build_step)