def inject_steps(self): # 'publish' step publish_step = PipelineStep( name='publish', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS, script_type=ScriptType.BOURNE_SHELL, ) publish_step.set_timeout(duration_string='4h') # 'prepare' step prepare_step = PipelineStep( name='prepare', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.BOURNE_SHELL, ) prepare_step.set_timeout(duration_string='30m') publish_step._add_dependency(prepare_step) yield prepare_step yield publish_step
def inject_steps(self): if self.trait.protecode(): self.image_scan_step = PipelineStep( name='scan_container_images', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, injected_by_trait=self.name, script_type=ScriptType.PYTHON3) self.image_scan_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.image_scan_step.set_timeout(duration_string='12h') yield self.image_scan_step if self.trait.clam_av(): self.malware_scan_step = PipelineStep( name='malware-scan', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, injected_by_trait=self.name, script_type=ScriptType.PYTHON3) self.malware_scan_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.malware_scan_step.set_timeout(duration_string='12h') yield self.malware_scan_step
def inject_steps(self): if self.trait.set_dependency_version_script_container_image(): privilege_mode = PrivilegeMode.PRIVILEGED else: privilege_mode = PrivilegeMode.UNPRIVILEGED # declare no dependencies --> run asap, but do not block other steps self.update_component_deps_step = PipelineStep( name='update_component_dependencies', raw_dict={ 'privilege_mode': privilege_mode, }, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3) self.update_component_deps_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.update_component_deps_step.set_timeout(duration_string='30m') for name, value in self.trait.vars().items(): self.update_component_deps_step.variables()[name] = value yield self.update_component_deps_step
class ReleaseTraitTransformer(TraitTransformer): name = 'release' def inject_steps(self): # inject 'release' step self.release_step = PipelineStep( name='release', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) yield self.release_step def process_pipeline_args(self, pipeline_args: 'JobVariant'): # we depend on all other steps for step in pipeline_args.steps(): self.release_step._add_dependency(step) # a 'release job' should only be triggered automatically if explicitly configured main_repo = pipeline_args.main_repository() if main_repo: if 'trigger' not in pipeline_args.raw['repo']: main_repo._trigger = False @classmethod def dependencies(cls): return {'version'} @classmethod def order_dependencies(cls): return {'publish'}
class VersionTraitTransformer(TraitTransformer): name = 'version' def inject_steps(self): self.version_step = PipelineStep( name='version', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, ) self.version_step.add_output(name=DIR_NAME, variable_name=ENV_VAR_NAME) self.version_step.set_timeout(duration_string='5m') yield self.version_step def process_pipeline_args(self, pipeline_args: JobVariant): # all steps depend from us and may consume our output for step in pipeline_args.steps(): if step == self.version_step: continue step._add_dependency(self.version_step) step.add_input(variable_name=ENV_VAR_NAME, name=DIR_NAME)
class DraftReleaseTraitTransformer(TraitTransformer): name = 'draft_release' def inject_steps(self): # inject 'release' step self.release_step = PipelineStep( name='create_draft_release_notes', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, pull_request_notification_policy=PullRequestNotificationPolicy.NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, ) self.release_step.set_timeout(duration_string='10m') yield self.release_step def process_pipeline_args(self, pipeline_args: JobVariant): cd_trait = pipeline_args.trait('component_descriptor') cd_step = pipeline_args.step(cd_trait.step_name()) self.release_step._add_dependency(cd_step) @classmethod def dependencies(cls): return {'version', 'component_descriptor'}
def inject_steps(self): self.image_scan_step = PipelineStep( name='scan_container_images', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.image_scan_step.add_input(*COMPONENT_DESCRIPTOR_DIR_INPUT) self.image_scan_step.set_timeout(duration_string='12h') yield self.image_scan_step
def inject_steps(self): self.meta_step = PipelineStep( name='meta', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) self.meta_step.add_output(name=DIR_NAME, variable_name=ENV_VAR_NAME) yield self.meta_step
def inject_steps(self): # inject 'release' step self.release_step = PipelineStep( name='release', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) yield self.release_step
def inject_steps(self): # declare no dependencies --> run asap, but do not block other steps rm_pr_label_step = PipelineStep( name='rm_pr_label', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) rm_pr_label_step.set_timeout(duration_string='5m') yield rm_pr_label_step
def inject_steps(self): # inject 'release' step self.release_step = PipelineStep( name='create_draft_release_notes', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) self.release_step.set_timeout(duration_string='10m') yield self.release_step
def inject_steps(self): # declare no dependencies --> run asap, but do not block other steps self.update_component_deps_step = PipelineStep( name='update_component_dependencies', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.update_component_deps_step.add_input( *COMPONENT_DESCRIPTOR_DIR_INPUT) self.update_component_deps_step.set_timeout(duration_string='30m') yield self.update_component_deps_step
def inject_steps(self): self.descriptor_step = PipelineStep( name=self.trait.step_name(), raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) self.descriptor_step.add_output(*COMPONENT_DESCRIPTOR_DIR_INPUT) self.descriptor_step.set_timeout(duration_string='30m') yield self.descriptor_step
def inject_steps(self): self.version_step = PipelineStep( name='version', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, injected_by_trait=self.name, script_type=ScriptType.PYTHON3, ) self.version_step.add_output(name=DIR_NAME, variable_name=ENV_VAR_NAME) self.version_step.set_timeout(duration_string='5m') yield self.version_step
def inject_steps(self): self.version_step = PipelineStep( name='version', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) self.version_step.add_output(name='version_path', variable_name='managed-version') self.version_step.set_timeout(duration_string='5m') yield self.version_step
def inject_steps(self): self.source_scan_step = PipelineStep( name='scan_sources', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.source_scan_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.source_scan_step.set_timeout(duration_string='18h') yield self.source_scan_step
def inject_steps(self): self.descriptor_step = PipelineStep( name=self.trait.step_name(), raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) self.descriptor_step.add_output( name=DIR_NAME, variable_name=ENV_VAR_NAME, ) self.descriptor_step.set_timeout(duration_string='30m') yield self.descriptor_step
class ImageScanTraitTransformer(TraitTransformer): name = 'image_scan' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) def inject_steps(self): self.image_scan_step = PipelineStep( name='scan_container_images', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) self.image_scan_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.image_scan_step.set_timeout(duration_string='12h') yield self.image_scan_step def process_pipeline_args(self, pipeline_args: JobVariant): # our step depends on dependency descriptor step component_descriptor_step = pipeline_args.step('component_descriptor') self.image_scan_step._add_dependency(component_descriptor_step) for trait_name in self.trait.trait_depends(): if not pipeline_args.has_trait(trait_name): raise ModelValidationError( f'dependency towards absent trait: {trait_name}') depended_on_trait = pipeline_args.trait(trait_name) # XXX refactor Trait/TraitTransformer transformer = depended_on_trait.transformer() # XXX step-injection may have (unintended) side-effects :-/ depended_on_step_names = { step.name for step in transformer.inject_steps() } for step in pipeline_args.steps(): if not step.name in depended_on_step_names: continue self.image_scan_step._add_dependency(step) # prevent cyclic dependencies (from auto-injected depends) if self.image_scan_step.name in step.depends(): step._remove_dependency(self.image_scan_step) @classmethod def dependencies(cls): return {'component_descriptor'} @classmethod def order_dependencies(cls): # required in case image-scanning should be done after publish # (-> auto-injected dependency for "prepare"-step towards _all_ steps) return {'publish'}
class ComponentDescriptorTraitTransformer(TraitTransformer): name = 'component_descriptor' def __init__(self, trait: ComponentDescriptorTrait, *args, **kwargs): super().__init__(*args, **kwargs) self.trait = not_none(trait) def inject_steps(self): self.descriptor_step = PipelineStep( name=self.trait.step_name(), raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3, ) self.descriptor_step.add_output( name=DIR_NAME, variable_name=ENV_VAR_NAME, ) self.descriptor_step.set_timeout(duration_string='30m') yield self.descriptor_step def process_pipeline_args(self, pipeline_args: 'JobVariant'): if pipeline_args.has_step('release'): release_step = pipeline_args.step('release') release_step.add_input( name=DIR_NAME, variable_name=ENV_VAR_NAME, ) # inject component_name if not configured if not self.trait.raw.get('component_name'): main_repo = pipeline_args.main_repository() component_name = '/'.join(( main_repo.repo_hostname(), main_repo.repo_path(), )) self.trait.raw['component_name'] = component_name @classmethod def dependencies(cls): return {'version'} @classmethod def order_dependencies(cls): # dependency is required, as we need to patch the 'release' step return {'release'}
def _create_build_step(self, name: str, step_dict: dict): return PipelineStep( name=name, is_synthetic=False, raw_dict=step_dict, script_type=ScriptType.BOURNE_SHELL, )
def _examinee(self, name='dontcare', **kwargs): return PipelineStep( name=name, is_synthetic=False, script_type=ScriptType.BOURNE_SHELL, raw_dict=kwargs, )
def pipeline_step(self, name, is_synthetic=False, **kwargs): return PipelineStep( name=name, is_synthetic=is_synthetic, script_type=ScriptType.BOURNE_SHELL, raw_dict=kwargs, )
class UpdateComponentDependenciesTraitTransformer(TraitTransformer): name = 'update_component_deps' def __init__(self, trait, *args, **kwargs): self.trait = trait super().__init__(*args, **kwargs) @classmethod def order_dependencies(cls): return {'component_descriptor'} @classmethod def dependencies(cls): return {'component_descriptor'} def inject_steps(self): if self.trait.set_dependency_version_script_container_image(): privilege_mode = PrivilegeMode.PRIVILEGED else: privilege_mode = PrivilegeMode.UNPRIVILEGED # declare no dependencies --> run asap, but do not block other steps self.update_component_deps_step = PipelineStep( name='update_component_dependencies', raw_dict={ 'privilege_mode': privilege_mode, }, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3) self.update_component_deps_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor. ENV_VAR_NAME, ) self.update_component_deps_step.set_timeout(duration_string='30m') for name, value in self.trait.vars().items(): self.update_component_deps_step.variables()[name] = value yield self.update_component_deps_step def process_pipeline_args(self, pipeline_args: JobVariant): # our step depends on dependendency descriptor step component_descriptor_step = pipeline_args.step( concourse.model.traits.component_descriptor. DEFAULT_COMPONENT_DESCRIPTOR_STEP_NAME) self.update_component_deps_step._add_dependency( component_descriptor_step) upstream_component_name = self.trait.upstream_component_name() if upstream_component_name: self.update_component_deps_step.variables( )['UPSTREAM_COMPONENT_NAME'] = '"{cn}"'.format( cn=upstream_component_name, )
def _create_build_step(self, name: str, step_dict: dict): return PipelineStep( name=name, is_synthetic=False, notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS, raw_dict=step_dict, script_type=ScriptType.BOURNE_SHELL, )
def _examinee(self, name='dontcare', **kwargs): return PipelineStep( name=name, is_synthetic=False, notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS, script_type=ScriptType.BOURNE_SHELL, raw_dict=kwargs, )
class ReleaseTraitTransformer(TraitTransformer): name = 'release' def __init__(self, trait: ReleaseTrait, *args, **kwargs): super().__init__(*args, **kwargs) self.trait = trait def inject_steps(self): if self.trait.release_callback_image_reference(): # we need privileged container in order to run callback in container privilege_mode = PrivilegeMode.PRIVILEGED else: privilege_mode = PrivilegeMode.UNPRIVILEGED # inject 'release' step self.release_step = PipelineStep( name='release', raw_dict={ 'privilege_mode': privilege_mode, }, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, ) yield self.release_step def process_pipeline_args(self, pipeline_args: JobVariant): # we depend on all other steps for step in pipeline_args.steps(): self.release_step._add_dependency(step) # a 'release job' should only be triggered automatically if explicitly configured main_repo = pipeline_args.main_repository() if main_repo: if 'trigger' not in pipeline_args.raw['repo']: main_repo._trigger = False @classmethod def dependencies(cls): return {'version', 'component_descriptor'} @classmethod def order_dependencies(cls): return {'publish'}
def inject_steps(self): self.image_alter_step = PipelineStep( name='alter_container_images', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3) yield self.image_alter_step
def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() self.meta_dir = os.path.join(self.tmp_dir.name, 'meta') os.mkdir(self.meta_dir) test_utils.populate_meta_dir(self.meta_dir) self.on_error_dir = os.path.join(self.tmp_dir.name, 'on_error_dir') os.mkdir(self.on_error_dir) self.job_step = PipelineStep( name='step1', is_synthetic=False, notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS, script_type=ScriptType.BOURNE_SHELL, raw_dict={}, ) self.job_step._notifications_cfg = NotificationCfgSet('default', {}) resource_registry = ResourceRegistry() meta_resource_identifier = ResourceIdentifier(type_name='meta', base_name='a_job') meta_resource = Resource(resource_identifier=meta_resource_identifier, raw_dict={}) resource_registry.add_resource(meta_resource) self.job_variant = JobVariant(name='a_job', raw_dict={}, resource_registry=resource_registry) # Set a main repository manually test_repo_logical_name = 'test-repository' self.job_variant._repos_dict = {} self.job_variant._repos_dict[ test_repo_logical_name] = RepositoryConfig( raw_dict={ 'branch': 'master', 'hostname': 'github.foo.bar', 'path': 'test/repo' }, logical_name=test_repo_logical_name, qualifier=None, is_main_repo=True) self.job_variant._main_repository_name = test_repo_logical_name self.job_variant._traits_dict = {} self.cfg_set = MagicMock() self.github_cfg = MagicMock() self.github_cfg.name = MagicMock(return_value='github_cfg') self.email_cfg = MagicMock() self.email_cfg.name = MagicMock(return_value='email_cfg') self.cfg_set.github = MagicMock(return_value=self.github_cfg) self.cfg_set.email = MagicMock(return_value=self.email_cfg) ctx_repo_mock = MagicMock(return_value='repo_url') ctx_repo_mock.base_url = MagicMock(return_value='repo_url') self.cfg_set.ctx_repository = MagicMock(return_value=ctx_repo_mock) self.render_step = step_def('notification') self.old_cwd = os.getcwd() os.chdir(self.tmp_dir.name)
def inject_steps(self): # declare no dependencies --> run asap, but do not block other steps self.update_component_deps_step = PipelineStep( name='update_component_dependencies', raw_dict={}, is_synthetic=True, notification_policy=StepNotificationPolicy.NO_NOTIFICATION, script_type=ScriptType.PYTHON3 ) self.update_component_deps_step.add_input( name=concourse.model.traits.component_descriptor.DIR_NAME, variable_name=concourse.model.traits.component_descriptor.ENV_VAR_NAME, ) self.update_component_deps_step.set_timeout(duration_string='30m') for name, value in self.trait.vars().items(): self.update_component_deps_step.variables()[name] = value yield self.update_component_deps_step
def inject_steps(self): if self.trait.release_callback_image_reference(): # we need privileged container in order to run callback in container privilege_mode = PrivilegeMode.PRIVILEGED else: privilege_mode = PrivilegeMode.UNPRIVILEGED # inject 'release' step self.release_step = PipelineStep( name='release', raw_dict={ 'privilege_mode': privilege_mode, }, is_synthetic=True, pull_request_notification_policy=PullRequestNotificationPolicy. NO_NOTIFICATION, injecting_trait_name=self.name, script_type=ScriptType.PYTHON3, ) yield self.release_step