def inject_steps(self):
# 'publish' step
publish_step = PipelineStep(
name='publish',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS,
script_type=ScriptType.BOURNE_SHELL,
)
publish_step.set_timeout(duration_string='4h')
# 'prepare' step
prepare_step = PipelineStep(
name='prepare',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.BOURNE_SHELL,
)
prepare_step.set_timeout(duration_string='30m')
publish_step._add_dependency(prepare_step)
yield prepare_step
yield publish_step
def inject_steps(self):
if self.trait.protecode():
self.image_scan_step = PipelineStep(
name='scan_container_images',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
injected_by_trait=self.name,
script_type=ScriptType.PYTHON3)
self.image_scan_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.
ENV_VAR_NAME,
)
self.image_scan_step.set_timeout(duration_string='12h')
yield self.image_scan_step
if self.trait.clam_av():
self.malware_scan_step = PipelineStep(
name='malware-scan',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
injected_by_trait=self.name,
script_type=ScriptType.PYTHON3)
self.malware_scan_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.
ENV_VAR_NAME,
)
self.malware_scan_step.set_timeout(duration_string='12h')
yield self.malware_scan_step
def inject_steps(self):
if self.trait.set_dependency_version_script_container_image():
privilege_mode = PrivilegeMode.PRIVILEGED
else:
privilege_mode = PrivilegeMode.UNPRIVILEGED
# declare no dependencies --> run asap, but do not block other steps
self.update_component_deps_step = PipelineStep(
name='update_component_dependencies',
raw_dict={
'privilege_mode': privilege_mode,
},
is_synthetic=True,
pull_request_notification_policy=PullRequestNotificationPolicy.
NO_NOTIFICATION,
injecting_trait_name=self.name,
script_type=ScriptType.PYTHON3)
self.update_component_deps_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.
ENV_VAR_NAME,
)
self.update_component_deps_step.set_timeout(duration_string='30m')
for name, value in self.trait.vars().items():
self.update_component_deps_step.variables()[name] = value
yield self.update_component_deps_step
class ReleaseTraitTransformer(TraitTransformer):
name = 'release'
def inject_steps(self):
# inject 'release' step
self.release_step = PipelineStep(
name='release',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
yield self.release_step
def process_pipeline_args(self, pipeline_args: 'JobVariant'):
# we depend on all other steps
for step in pipeline_args.steps():
self.release_step._add_dependency(step)
# a 'release job' should only be triggered automatically if explicitly configured
main_repo = pipeline_args.main_repository()
if main_repo:
if 'trigger' not in pipeline_args.raw['repo']:
main_repo._trigger = False
@classmethod
def dependencies(cls):
return {'version'}
@classmethod
def order_dependencies(cls):
return {'publish'}
class VersionTraitTransformer(TraitTransformer):
name = 'version'
def inject_steps(self):
self.version_step = PipelineStep(
name='version',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
pull_request_notification_policy=PullRequestNotificationPolicy.
NO_NOTIFICATION,
injecting_trait_name=self.name,
script_type=ScriptType.PYTHON3,
)
self.version_step.add_output(name=DIR_NAME, variable_name=ENV_VAR_NAME)
self.version_step.set_timeout(duration_string='5m')
yield self.version_step
def process_pipeline_args(self, pipeline_args: JobVariant):
# all steps depend from us and may consume our output
for step in pipeline_args.steps():
if step == self.version_step:
continue
step._add_dependency(self.version_step)
step.add_input(variable_name=ENV_VAR_NAME, name=DIR_NAME)
class DraftReleaseTraitTransformer(TraitTransformer):
name = 'draft_release'
def inject_steps(self):
# inject 'release' step
self.release_step = PipelineStep(
name='create_draft_release_notes',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
pull_request_notification_policy=PullRequestNotificationPolicy.NO_NOTIFICATION,
injecting_trait_name=self.name,
script_type=ScriptType.PYTHON3,
)
self.release_step.set_timeout(duration_string='10m')
yield self.release_step
def process_pipeline_args(self, pipeline_args: JobVariant):
cd_trait = pipeline_args.trait('component_descriptor')
cd_step = pipeline_args.step(cd_trait.step_name())
self.release_step._add_dependency(cd_step)
@classmethod
def dependencies(cls):
return {'version', 'component_descriptor'}
def inject_steps(self):
self.image_scan_step = PipelineStep(
name='scan_container_images',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3)
self.image_scan_step.add_input(*COMPONENT_DESCRIPTOR_DIR_INPUT)
self.image_scan_step.set_timeout(duration_string='12h')
yield self.image_scan_step
def inject_steps(self):
self.meta_step = PipelineStep(
name='meta',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
self.meta_step.add_output(name=DIR_NAME, variable_name=ENV_VAR_NAME)
yield self.meta_step
def inject_steps(self):
# inject 'release' step
self.release_step = PipelineStep(
name='release',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
yield self.release_step
def inject_steps(self):
# declare no dependencies --> run asap, but do not block other steps
rm_pr_label_step = PipelineStep(
name='rm_pr_label',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3)
rm_pr_label_step.set_timeout(duration_string='5m')
yield rm_pr_label_step
def inject_steps(self):
# inject 'release' step
self.release_step = PipelineStep(
name='create_draft_release_notes',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
self.release_step.set_timeout(duration_string='10m')
yield self.release_step
def inject_steps(self):
# declare no dependencies --> run asap, but do not block other steps
self.update_component_deps_step = PipelineStep(
name='update_component_dependencies',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3)
self.update_component_deps_step.add_input(
*COMPONENT_DESCRIPTOR_DIR_INPUT)
self.update_component_deps_step.set_timeout(duration_string='30m')
yield self.update_component_deps_step
def inject_steps(self):
self.descriptor_step = PipelineStep(
name=self.trait.step_name(),
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
self.descriptor_step.add_output(*COMPONENT_DESCRIPTOR_DIR_INPUT)
self.descriptor_step.set_timeout(duration_string='30m')
yield self.descriptor_step
def inject_steps(self):
self.version_step = PipelineStep(
name='version',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
injected_by_trait=self.name,
script_type=ScriptType.PYTHON3,
)
self.version_step.add_output(name=DIR_NAME, variable_name=ENV_VAR_NAME)
self.version_step.set_timeout(duration_string='5m')
yield self.version_step
def inject_steps(self):
self.version_step = PipelineStep(
name='version',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
self.version_step.add_output(name='version_path',
variable_name='managed-version')
self.version_step.set_timeout(duration_string='5m')
yield self.version_step
def inject_steps(self):
self.source_scan_step = PipelineStep(
name='scan_sources',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3)
self.source_scan_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.
ENV_VAR_NAME,
)
self.source_scan_step.set_timeout(duration_string='18h')
yield self.source_scan_step
def inject_steps(self):
self.descriptor_step = PipelineStep(
name=self.trait.step_name(),
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
self.descriptor_step.add_output(
name=DIR_NAME,
variable_name=ENV_VAR_NAME,
)
self.descriptor_step.set_timeout(duration_string='30m')
yield self.descriptor_step
class ImageScanTraitTransformer(TraitTransformer):
name = 'image_scan'
def __init__(self, trait, *args, **kwargs):
self.trait = trait
super().__init__(*args, **kwargs)
def inject_steps(self):
self.image_scan_step = PipelineStep(
name='scan_container_images',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3)
self.image_scan_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.
ENV_VAR_NAME,
)
self.image_scan_step.set_timeout(duration_string='12h')
yield self.image_scan_step
def process_pipeline_args(self, pipeline_args: JobVariant):
# our step depends on dependency descriptor step
component_descriptor_step = pipeline_args.step('component_descriptor')
self.image_scan_step._add_dependency(component_descriptor_step)
for trait_name in self.trait.trait_depends():
if not pipeline_args.has_trait(trait_name):
raise ModelValidationError(
f'dependency towards absent trait: {trait_name}')
depended_on_trait = pipeline_args.trait(trait_name)
# XXX refactor Trait/TraitTransformer
transformer = depended_on_trait.transformer()
# XXX step-injection may have (unintended) side-effects :-/
depended_on_step_names = {
step.name
for step in transformer.inject_steps()
}
for step in pipeline_args.steps():
if not step.name in depended_on_step_names:
continue
self.image_scan_step._add_dependency(step)
# prevent cyclic dependencies (from auto-injected depends)
if self.image_scan_step.name in step.depends():
step._remove_dependency(self.image_scan_step)
@classmethod
def dependencies(cls):
return {'component_descriptor'}
@classmethod
def order_dependencies(cls):
# required in case image-scanning should be done after publish
# (-> auto-injected dependency for "prepare"-step towards _all_ steps)
return {'publish'}
class ComponentDescriptorTraitTransformer(TraitTransformer):
name = 'component_descriptor'
def __init__(self, trait: ComponentDescriptorTrait, *args, **kwargs):
super().__init__(*args, **kwargs)
self.trait = not_none(trait)
def inject_steps(self):
self.descriptor_step = PipelineStep(
name=self.trait.step_name(),
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3,
)
self.descriptor_step.add_output(
name=DIR_NAME,
variable_name=ENV_VAR_NAME,
)
self.descriptor_step.set_timeout(duration_string='30m')
yield self.descriptor_step
def process_pipeline_args(self, pipeline_args: 'JobVariant'):
if pipeline_args.has_step('release'):
release_step = pipeline_args.step('release')
release_step.add_input(
name=DIR_NAME,
variable_name=ENV_VAR_NAME,
)
# inject component_name if not configured
if not self.trait.raw.get('component_name'):
main_repo = pipeline_args.main_repository()
component_name = '/'.join((
main_repo.repo_hostname(),
main_repo.repo_path(),
))
self.trait.raw['component_name'] = component_name
@classmethod
def dependencies(cls):
return {'version'}
@classmethod
def order_dependencies(cls):
# dependency is required, as we need to patch the 'release' step
return {'release'}
def _create_build_step(self, name: str, step_dict: dict):
return PipelineStep(
name=name,
is_synthetic=False,
raw_dict=step_dict,
script_type=ScriptType.BOURNE_SHELL,
)
def _examinee(self, name='dontcare', **kwargs):
return PipelineStep(
name=name,
is_synthetic=False,
script_type=ScriptType.BOURNE_SHELL,
raw_dict=kwargs,
)
def pipeline_step(self, name, is_synthetic=False, **kwargs):
return PipelineStep(
name=name,
is_synthetic=is_synthetic,
script_type=ScriptType.BOURNE_SHELL,
raw_dict=kwargs,
)
class UpdateComponentDependenciesTraitTransformer(TraitTransformer):
name = 'update_component_deps'
def __init__(self, trait, *args, **kwargs):
self.trait = trait
super().__init__(*args, **kwargs)
@classmethod
def order_dependencies(cls):
return {'component_descriptor'}
@classmethod
def dependencies(cls):
return {'component_descriptor'}
def inject_steps(self):
if self.trait.set_dependency_version_script_container_image():
privilege_mode = PrivilegeMode.PRIVILEGED
else:
privilege_mode = PrivilegeMode.UNPRIVILEGED
# declare no dependencies --> run asap, but do not block other steps
self.update_component_deps_step = PipelineStep(
name='update_component_dependencies',
raw_dict={
'privilege_mode': privilege_mode,
},
is_synthetic=True,
pull_request_notification_policy=PullRequestNotificationPolicy.
NO_NOTIFICATION,
injecting_trait_name=self.name,
script_type=ScriptType.PYTHON3)
self.update_component_deps_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.
ENV_VAR_NAME,
)
self.update_component_deps_step.set_timeout(duration_string='30m')
for name, value in self.trait.vars().items():
self.update_component_deps_step.variables()[name] = value
yield self.update_component_deps_step
def process_pipeline_args(self, pipeline_args: JobVariant):
# our step depends on dependendency descriptor step
component_descriptor_step = pipeline_args.step(
concourse.model.traits.component_descriptor.
DEFAULT_COMPONENT_DESCRIPTOR_STEP_NAME)
self.update_component_deps_step._add_dependency(
component_descriptor_step)
upstream_component_name = self.trait.upstream_component_name()
if upstream_component_name:
self.update_component_deps_step.variables(
)['UPSTREAM_COMPONENT_NAME'] = '"{cn}"'.format(
cn=upstream_component_name, )
def _create_build_step(self, name: str, step_dict: dict):
return PipelineStep(
name=name,
is_synthetic=False,
notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS,
raw_dict=step_dict,
script_type=ScriptType.BOURNE_SHELL,
)
def _examinee(self, name='dontcare', **kwargs):
return PipelineStep(
name=name,
is_synthetic=False,
notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS,
script_type=ScriptType.BOURNE_SHELL,
raw_dict=kwargs,
)
class ReleaseTraitTransformer(TraitTransformer):
name = 'release'
def __init__(self, trait: ReleaseTrait, *args, **kwargs):
super().__init__(*args, **kwargs)
self.trait = trait
def inject_steps(self):
if self.trait.release_callback_image_reference():
# we need privileged container in order to run callback in container
privilege_mode = PrivilegeMode.PRIVILEGED
else:
privilege_mode = PrivilegeMode.UNPRIVILEGED
# inject 'release' step
self.release_step = PipelineStep(
name='release',
raw_dict={
'privilege_mode': privilege_mode,
},
is_synthetic=True,
pull_request_notification_policy=PullRequestNotificationPolicy.
NO_NOTIFICATION,
injecting_trait_name=self.name,
script_type=ScriptType.PYTHON3,
)
yield self.release_step
def process_pipeline_args(self, pipeline_args: JobVariant):
# we depend on all other steps
for step in pipeline_args.steps():
self.release_step._add_dependency(step)
# a 'release job' should only be triggered automatically if explicitly configured
main_repo = pipeline_args.main_repository()
if main_repo:
if 'trigger' not in pipeline_args.raw['repo']:
main_repo._trigger = False
@classmethod
def dependencies(cls):
return {'version', 'component_descriptor'}
@classmethod
def order_dependencies(cls):
return {'publish'}
def inject_steps(self):
self.image_alter_step = PipelineStep(
name='alter_container_images',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3)
yield self.image_alter_step
def setUp(self):
self.tmp_dir = tempfile.TemporaryDirectory()
self.meta_dir = os.path.join(self.tmp_dir.name, 'meta')
os.mkdir(self.meta_dir)
test_utils.populate_meta_dir(self.meta_dir)
self.on_error_dir = os.path.join(self.tmp_dir.name, 'on_error_dir')
os.mkdir(self.on_error_dir)
self.job_step = PipelineStep(
name='step1',
is_synthetic=False,
notification_policy=StepNotificationPolicy.NOTIFY_PULL_REQUESTS,
script_type=ScriptType.BOURNE_SHELL,
raw_dict={},
)
self.job_step._notifications_cfg = NotificationCfgSet('default', {})
resource_registry = ResourceRegistry()
meta_resource_identifier = ResourceIdentifier(type_name='meta',
base_name='a_job')
meta_resource = Resource(resource_identifier=meta_resource_identifier,
raw_dict={})
resource_registry.add_resource(meta_resource)
self.job_variant = JobVariant(name='a_job',
raw_dict={},
resource_registry=resource_registry)
# Set a main repository manually
test_repo_logical_name = 'test-repository'
self.job_variant._repos_dict = {}
self.job_variant._repos_dict[
test_repo_logical_name] = RepositoryConfig(
raw_dict={
'branch': 'master',
'hostname': 'github.foo.bar',
'path': 'test/repo'
},
logical_name=test_repo_logical_name,
qualifier=None,
is_main_repo=True)
self.job_variant._main_repository_name = test_repo_logical_name
self.job_variant._traits_dict = {}
self.cfg_set = MagicMock()
self.github_cfg = MagicMock()
self.github_cfg.name = MagicMock(return_value='github_cfg')
self.email_cfg = MagicMock()
self.email_cfg.name = MagicMock(return_value='email_cfg')
self.cfg_set.github = MagicMock(return_value=self.github_cfg)
self.cfg_set.email = MagicMock(return_value=self.email_cfg)
ctx_repo_mock = MagicMock(return_value='repo_url')
ctx_repo_mock.base_url = MagicMock(return_value='repo_url')
self.cfg_set.ctx_repository = MagicMock(return_value=ctx_repo_mock)
self.render_step = step_def('notification')
self.old_cwd = os.getcwd()
os.chdir(self.tmp_dir.name)
def inject_steps(self):
# declare no dependencies --> run asap, but do not block other steps
self.update_component_deps_step = PipelineStep(
name='update_component_dependencies',
raw_dict={},
is_synthetic=True,
notification_policy=StepNotificationPolicy.NO_NOTIFICATION,
script_type=ScriptType.PYTHON3
)
self.update_component_deps_step.add_input(
name=concourse.model.traits.component_descriptor.DIR_NAME,
variable_name=concourse.model.traits.component_descriptor.ENV_VAR_NAME,
)
self.update_component_deps_step.set_timeout(duration_string='30m')
for name, value in self.trait.vars().items():
self.update_component_deps_step.variables()[name] = value
yield self.update_component_deps_step
def inject_steps(self):
if self.trait.release_callback_image_reference():
# we need privileged container in order to run callback in container
privilege_mode = PrivilegeMode.PRIVILEGED
else:
privilege_mode = PrivilegeMode.UNPRIVILEGED
# inject 'release' step
self.release_step = PipelineStep(
name='release',
raw_dict={
'privilege_mode': privilege_mode,
},
is_synthetic=True,
pull_request_notification_policy=PullRequestNotificationPolicy.
NO_NOTIFICATION,
injecting_trait_name=self.name,
script_type=ScriptType.PYTHON3,
)
yield self.release_step
