def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
"""
pipeline = configurator.ensure_pipeline_group(
config['pipeline_group']
).ensure_replacement_of_pipeline(
config['pipeline_name']
).ensure_material(
GitMaterial(
'https://github.com/edx/edx-gomatic',
polling=True,
material_name='edx-gomatic',
destination_directory='edx-gomatic',
)
).ensure_material(
GitMaterial(
'[email protected]:edx-ops/gomatic-secure.git',
polling=True,
destination_directory='gomatic-secure',
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX
)
).set_label_template('${edx-gomatic[:7]}')
pipeline.ensure_encrypted_environment_variables(
{
'GOMATIC_USER': config['gomatic_user'],
'GOMATIC_PASSWORD': config['gomatic_password']
}
)
stage = pipeline.ensure_stage('deploy_gomatic_stage')
job = stage.ensure_job('deploy_gomatic_scripts_job')
tasks.generate_requirements_install(job, 'edx-gomatic')
job.add_task(
ExecTask(
[
'/usr/bin/python',
'./deploy_pipelines.py',
'-v',
'tools',
'-f',
'config.yml'
],
working_dir='edx-gomatic'
)
)
def __enter__(self):
try:
start_go_server(self.gocd_version,
self.gocd_download_version_string)
configurator = GoCdConfigurator(HostRestClient('localhost:8153'))
pipeline = configurator \
.ensure_pipeline_group("P.Group") \
.ensure_replacement_of_pipeline("more-options") \
.set_timer("0 15 22 * * ?") \
.set_git_material(GitMaterial("https://github.com/SpringerSBM/gomatic.git", material_name="some-material-name", polling=False)) \
.ensure_environment_variables({'JAVA_HOME': '/opt/java/jdk-1.7'}) \
.ensure_parameters({'environment': 'qa'})
stage = pipeline.ensure_stage("earlyStage")
job = stage.ensure_job("earlyWorm").ensure_artifacts({
Artifact.get_build_artifact("scripts/*", "files"),
Artifact.get_build_artifact("target/universal/myapp*.zip",
"artifacts"),
Artifact.get_test_artifact("from", "to")
}).set_runs_on_all_agents()
job.add_task(ExecTask(['ls']))
configurator.save_updated_config(save_config_locally=True)
return GoCdConfigurator(HostRestClient('localhost:8153'))
except:
# Swallow exception if __exit__ returns a True value
if self.__exit__(*sys.exc_info()):
pass
else:
raise
def test_all_versions(self):
for gocd_version, gocd_download_version_string in self.gocd_versions:
print 'test_all_versions', "*" * 60, gocd_version
with populated_go_server(
gocd_version,
gocd_download_version_string) as configurator:
self.assertEquals(
["P.Group"],
[p.name for p in configurator.pipeline_groups])
self.assertEquals(["more-options"], [
p.name for p in configurator.pipeline_groups[0].pipelines
])
pipeline = configurator.pipeline_groups[0].pipelines[0]
self.assertEquals("0 15 22 * * ?", pipeline.timer)
self.assertEquals(
GitMaterial("https://github.com/SpringerSBM/gomatic.git",
material_name="some-material-name",
polling=False), pipeline.git_material)
self.assertEquals({'JAVA_HOME': '/opt/java/jdk-1.7'},
pipeline.environment_variables)
self.assertEquals({'environment': 'qa'}, pipeline.parameters)
self.assertEquals(['earlyStage'],
[s.name for s in pipeline.stages])
self.assertEquals(['earlyWorm'],
[j.name for j in pipeline.stages[0].jobs])
job = pipeline.stages[0].jobs[0]
self.assertEquals(
{
Artifact.get_build_artifact("scripts/*", "files"),
Artifact.get_build_artifact(
"target/universal/myapp*.zip", "artifacts"),
Artifact.get_test_artifact("from", "to")
}, job.artifacts)
self.assertEquals(True, job.runs_on_all_agents)
self.assertEquals([ExecTask(['ls'])], job.tasks)
def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- pipeline_group
- pipeline_name
- asgard_api_endpoints
- asgard_token
- aws_access_key_id
- aws_secret_access_key
- cron_timer
"""
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])\
.set_timer(config['cron_timer'])\
.set_git_material(GitMaterial(
"https://github.com/edx/tubular.git",
polling=True,
destination_directory="tubular",
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX
))
stages.generate_asg_cleanup(pipeline, config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'])
def ignore_test_can_save_multiple_times_using_same_configurator(self):
gocd_version, gocd_download_version_string = self.gocd_versions[-1]
print('test_can_save_multiple_times_using_same_configurator', "*" * 60,
gocd_version)
with populated_go_server(gocd_version,
gocd_download_version_string) as configurator:
pipeline = configurator \
.ensure_pipeline_group("Test") \
.ensure_replacement_of_pipeline("new-one")
pipeline.set_git_material(
GitMaterial("https://github.com/SpringerSBM/gomatic.git",
polling=False))
job = pipeline.ensure_stage("build").ensure_job("build")
job.ensure_task(ExecTask(["ls"]))
configurator.save_updated_config(save_config_locally=True,
dry_run=False)
pipeline = configurator \
.ensure_pipeline_group("Test") \
.ensure_replacement_of_pipeline("new-two")
pipeline.set_git_material(
GitMaterial("https://github.com/SpringerSBM/gomatic.git",
polling=False))
job = pipeline.ensure_stage("build").ensure_job("build")
job.ensure_task(ExecTask(["ls"]))
configurator.save_updated_config(save_config_locally=True,
dry_run=False)
self.assertEquals(
1,
len(
configurator.ensure_pipeline_group('Test').find_pipeline(
'new-one').stages))
self.assertEquals(
1,
len(
configurator.ensure_pipeline_group('Test').find_pipeline(
'new-two').stages))
def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
materials: List of dictionaries of the materials used in this pipeline
upstream_pipelines: List of dictionaries of the upstream piplines that feed in to the rollback pipeline.
"""
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])\
.ensure_environment_variables({'WAIT_SLEEP_TIME': config['tubular_sleep_wait_time']})
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=set(material['ignore_patterns'])))
# Specify the upstream deploy pipeline material for this rollback pipeline.
# Assumes there's only a single upstream pipeline material for this pipeline.
rollback_material = config['upstream_pipeline']
pipeline.ensure_material(
PipelineMaterial(pipeline_name=rollback_material['pipeline_name'],
stage_name=rollback_material['stage_name'],
material_name=rollback_material['material_name']))
# Specify the artifact that will be fetched containing the previous deployment information.
# Assumes there's only a single upstream artifact used by this pipeline.
artifact_config = config['upstream_deploy_artifact']
deploy_file_location = utils.ArtifactLocation(
artifact_config['pipeline_name'], artifact_config['stage_name'],
artifact_config['job_name'], artifact_config['artifact_name'])
# Create the armed stage as this pipeline needs to auto-execute
stages.generate_armed_stage(pipeline, constants.ARMED_JOB_NAME)
# Create a single stage in the pipeline which will rollback to the previous ASGs/AMI.
rollback_stage = stages.generate_rollback_asg_stage(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['hipchat_token'],
constants.HIPCHAT_ROOM,
deploy_file_location,
)
# Since we only want this stage to rollback via manual approval, ensure that it is set on this stage.
rollback_stage.set_has_manual_approval()
def deployment_internal(deployment,
branch=None,
polling=True,
destination_directory=None,
ignore_patterns=frozenset()):
"""
Initialize a GitMaterial representing a deployment's internal configuration repo.
Args:
deployment (str): Deployment for which to create the material (e.g., 'edx', 'edge')
Returns:
GitMaterial
"""
return GitMaterial(
url='[email protected]:edx/{}-internal.git'.format(deployment),
branch=branch,
polling=polling,
destination_directory=destination_directory
or '{}-internal'.format(deployment),
ignore_patterns=ignore_patterns or constants.MATERIAL_IGNORE_ALL_REGEX,
shallow=True,
)
def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
Optional variables:
- configuration_secure_version
- configuration_internal_version
"""
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])
# Example materials yaml
# materials:
# - url: "https://github.com/edx/tubular"
# branch: "release"
# material_name: "tubular"
# polling: "True"
# destination_directory: "tubular"
# ignore_patterns:
# - '**/*'
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=set(material['ignore_patterns'])))
# If no upstream pipelines exist, don't install them!
for material in config.get('upstream_pipelines', []):
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
#
# Create the AMI-building stage.
#
stages.generate_launch_instance(
pipeline,
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['ec2_vpc_subnet_id'],
config['ec2_security_group_id'],
config['ec2_instance_profile_name'],
config['base_ami_id'],
manual_approval=not config.get('auto_run', False))
stages.generate_run_play(
pipeline,
'playbooks/edx-east/edxapp.yml',
edp=utils.EDP(config['edx_environment'], config['edx_deployment'],
config['play_name']),
private_github_key=config['github_private_key'],
app_repo=config['app_repo'],
configuration_secure_dir='{}-secure'.format(config['edx_deployment']),
configuration_internal_dir='{}-internal'.format(
config['edx_deployment']),
hipchat_token=config['hipchat_token'],
hipchat_room='release',
edx_platform_version='$GO_REVISION_EDX_PLATFORM',
edx_platform_repo='$APP_REPO',
configuration_version='$GO_REVISION_CONFIGURATION',
edxapp_theme_source_repo=config['theme_url'],
edxapp_theme_version='$GO_REVISION_EDX_THEME',
edxapp_theme_name='$EDXAPP_THEME_NAME',
disable_edx_services='true',
COMMON_TAG_EC2_INSTANCE='true',
cache_id='$GO_PIPELINE_COUNTER')
configuration_secure_repo = config['{}_configuration_secure_repo'.format(
config['edx_deployment'])]
configuration_internal_repo = config[
'{}_configuration_internal_repo'.format(config['edx_deployment'])]
configuration_secure_version = '$GO_REVISION_{}_SECURE'.format(
config['edx_deployment'].upper())
configuration_internal_version = '$GO_REVISION_{}_INTERNAL'.format(
config['edx_deployment'].upper())
stages.generate_create_ami_from_instance(
pipeline,
edp=utils.EDP(config['edx_environment'], config['edx_deployment'],
config['play_name']),
app_repo=config['app_repo'],
app_version='$GO_REVISION_EDX_PLATFORM',
hipchat_token=config['hipchat_token'],
hipchat_room='release pipeline',
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
version_tags={
'configuration':
(config['configuration_url'], '$GO_REVISION_CONFIGURATION'),
'configuration_secure':
(configuration_secure_repo, configuration_secure_version),
'configuration_internal':
(configuration_internal_repo, configuration_internal_version),
'edxapp_theme':
(config['theme_url'], '$GO_REVISION_EDX_MICROSITE'),
})
#
# Create the DB migration running stage.
#
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME,
constants.ANSIBLE_INVENTORY_FILENAME)
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.KEY_PEM_FILENAME)
launch_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
for sub_app in ['cms', 'lms']:
stages.generate_run_migrations(
pipeline,
db_migration_pass=config['db_migration_pass'],
inventory_location=ansible_inventory_location,
instance_key_location=instance_ssh_key_location,
launch_info_location=launch_info_location,
application_user=config['db_migration_user'],
application_name=config['play_name'],
application_path=config['application_path'],
sub_application_name=sub_app)
#
# Create the stage to deploy the AMI.
#
ami_file_location = utils.ArtifactLocation(pipeline.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
'ami.yml')
stages.generate_deploy_ami(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
ami_file_location,
manual_approval=not config.get('auto_deploy_ami', False))
#
# Create the stage to terminate the EC2 instance used to both build the AMI and run DB migrations.
#
instance_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
stages.generate_terminate_instance(
pipeline,
instance_info_location,
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_token=config['hipchat_token'],
runif='any')
def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
materials: A list of dictionaries of the materials used in this pipeline
upstream_pipelines: a list of dictionaries of the upstream pipelines that feed in to the manual verification
"""
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=set(material['ignore_patterns'])))
for material in config['upstream_pipelines']:
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
# What this accomplishes:
# When a pipeline such as edx stage runs this pipeline is downstream. Since the first stage is automatic
# the git materials will be carried over from the first pipeline.
#
# The second stage in this pipeline requires manual approval.
#
# This allows the overall workflow to remain paused while manual verification is completed and allows the git
# materials to stay pinned.
#
# Once the second phase is approved, the workflow will continue and pipelines downstream will continue to execute
# with the same pinned materials from the upstream pipeline.
stages.generate_armed_stage(pipeline,
constants.INITIAL_VERIFICATION_STAGE_NAME)
# For now, you can only trigger builds on a single jenkins server, because you can only
# define a single username/token.
# And all the jobs that you want to trigger need the same job token defined.
# TODO: refactor when required so that each job can define their own user and job tokens
pipeline.ensure_unencrypted_secure_environment_variables({
'JENKINS_USER_TOKEN':
config['jenkins_user_token'],
'JENKINS_JOB_TOKEN':
config['jenkins_job_token']
})
# Create the stage with the Jenkins jobs
jenkins_stage = pipeline.ensure_stage(
constants.JENKINS_VERIFICATION_STAGE_NAME)
jenkins_stage.set_has_manual_approval()
jenkins_user_name = config['jenkins_user_name']
for jenkins in config['jenkins_verifications']:
pipeline_job_name = jenkins['pipeline_job_name']
jenkins_url = jenkins['url']
jenkins_job_name = jenkins['job_name']
key, _, param = jenkins['param'].partition(' ')
jenkins_param = {key: param}
job = jenkins_stage.ensure_job(pipeline_job_name)
tasks.generate_package_install(job, 'tubular')
tasks.trigger_jenkins_build(job, jenkins_url, jenkins_user_name,
jenkins_job_name, jenkins_param)
manual_verification_stage = pipeline.ensure_stage(
constants.MANUAL_VERIFICATION_STAGE_NAME)
manual_verification_stage.set_has_manual_approval()
manual_verification_job = manual_verification_stage.ensure_job(
constants.MANUAL_VERIFICATION_JOB_NAME)
manual_verification_job.add_task(
ExecTask([
'/bin/bash', '-c',
'echo Manual Verification run number $GO_PIPELINE_COUNTER completed by $GO_TRIGGER_USER'
], ))
def install_pipelines(configurator, config):
"""
Install pipelines that can build the edX api-gateway.
"""
pipeline = configurator \
.ensure_pipeline_group(config['pipeline']['group']) \
.ensure_replacement_of_pipeline(config['pipeline']['name']) \
.set_label_template('${api-manager}') \
.set_git_material(GitMaterial(
config['github']['server_uri'] + '/' + config['github']['repository'],
branch='#{GIT_BRANCH}',
material_name='api-manager',
destination_directory=API_MANAGER_WORKING_DIR
))
pipeline.ensure_parameters({'GIT_BRANCH': config['github']['branch']})
pipeline.ensure_environment_variables({
'SWAGGER_CODEGEN_JAR':
config['swagger_codegen_jar'],
'GITHUB_API_REPO':
config['github']['repository'],
'GITHUB_API_URI':
config['github']['api_uri'],
'GITHUB_API_POLL_WAIT_S':
config['github']['api_poll_wait_s'],
'GITHUB_API_POLL_RETRIES':
config['github']['api_poll_retries']
})
# Note, need to move this Github poll hack to something less of a hack at some point.
setup_stage = pipeline.ensure_stage(SETUP_STAGE_NAME)
wait_for_travis_job = setup_stage.ensure_job(WAIT_FOR_TRAVIS_JOB_NAME)
# pylint: disable=bad-continuation
wait_for_travis_job.add_task(
ExecTask([
'/bin/bash', '-c', 'i=0; until python -c "'
'import requests; '
'assert(requests.get('
'\'${GITHUB_API_URI}/${GITHUB_API_REPO}/commits/{}/status\'.format('
'\'${GO_REVISION_API_MANAGER}\''
')'
').json()[\'state\'] == \'success\')'
'"; '
'do i=$((i+1)); '
'if [ $i -gt ${GITHUB_API_POLL_RETRIES} ]; '
'then exit 1; '
'fi; '
'sleep ${GITHUB_API_POLL_WAIT_S}; '
'done'
]))
# pylint: enable=bad-continuation
download_stage = pipeline.ensure_stage(
DOWNLOAD_STAGE_NAME).set_clean_working_dir()
swagger_codegen_job = download_stage.ensure_job(
SWAGGER_CODEGEN_JOB_NAME).ensure_artifacts(
{BuildArtifact(SWAGGER_JAR)})
swagger_codegen_job.add_task(
ExecTask([
'/bin/bash', '-c',
'curl ${{SWAGGER_CODEGEN_JAR}} -o {swagger_jar}'.format(
swagger_jar=SWAGGER_JAR)
]))
build_stage = pipeline.ensure_stage(
BUILD_STAGE_NAME).set_clean_working_dir()
swagger_flatten_job = build_stage.ensure_job(
SWAGGER_FLATTEN_JOB_NAME).ensure_artifacts({
BuildArtifact('api-manager/swagger-build-artifacts/swagger.json')
})
artifact_params = {
'pipeline': pipeline.name,
'stage': DOWNLOAD_STAGE_NAME,
'job': SWAGGER_CODEGEN_JOB_NAME,
'src': FetchArtifactFile(SWAGGER_JAR),
'dest': API_MANAGER_WORKING_DIR
}
swagger_flatten_job.add_task(FetchArtifactTask(**artifact_params))
swagger_flatten_job.add_task(
ExecTask(['make', 'build'], working_dir=API_MANAGER_WORKING_DIR))
package_source_job = build_stage.ensure_job(
PACKAGE_SOURCE_JOB_NAME).ensure_artifacts(
{BuildArtifact('api-manager')})
package_source_job.add_task(
ExecTask([
'/bin/bash', '-c',
'pip install -t python-libs -r requirements/base.txt'
],
working_dir=API_MANAGER_WORKING_DIR))