def generate_migrate_stages(pipeline, config):
#
# Create the DB migration running stage.
#
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'ansible_inventory')
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'key.pem')
launch_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
for sub_app in config['edxapp_subapps']:
stages.generate_run_migrations(
pipeline,
db_migration_pass=config['db_migration_pass'],
inventory_location=ansible_inventory_location,
instance_key_location=instance_ssh_key_location,
launch_info_location=launch_info_location,
application_user=config['db_migration_user'],
application_name=config['play_name'],
application_path=config['application_path'],
sub_application_name=sub_app)
return pipeline
def generate_migrate_stages(pipeline, config):
"""
Generate stages to manage the migration of an environment,
and add them to ``pipeline``.
Required Config Values:
db_migration_pass
migration_duration_threshold
db_migration_user
play_name
application_path
alert_from_address
alert_to_addresses
"""
#
# Create the DB migration running stage.
#
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME,
constants.ANSIBLE_INVENTORY_FILENAME)
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.KEY_PEM_FILENAME)
launch_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
# Check the migration duration on the stage environment only.
if pipeline.name.startswith('STAGE'):
duration_threshold = config['migration_duration_threshold']
else:
duration_threshold = None
for sub_app in EDXAPP_SUBAPPS:
stages.generate_run_migrations(
pipeline,
db_migration_pass=config['db_migration_pass'],
inventory_location=ansible_inventory_location,
instance_key_location=instance_ssh_key_location,
launch_info_location=launch_info_location,
application_user=config['db_migration_user'],
application_name=config['play_name'],
application_path=config['application_path'],
sub_application_name=sub_app,
duration_threshold=duration_threshold,
from_address=config['alert_from_address'],
to_addresses=config['alert_to_addresses'])
return pipeline
def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
materials: List of dictionaries of the materials used in this pipeline
upstream_pipelines: List of dictionaries of the upstream piplines that feed in to the rollback pipeline.
"""
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])\
.ensure_environment_variables({'WAIT_SLEEP_TIME': config['tubular_sleep_wait_time']})
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=set(material['ignore_patterns'])))
# Specify the upstream deploy pipeline material for this rollback pipeline.
# Assumes there's only a single upstream pipeline material for this pipeline.
rollback_material = config['upstream_pipeline']
pipeline.ensure_material(
PipelineMaterial(pipeline_name=rollback_material['pipeline_name'],
stage_name=rollback_material['stage_name'],
material_name=rollback_material['material_name']))
# Specify the artifact that will be fetched containing the previous deployment information.
# Assumes there's only a single upstream artifact used by this pipeline.
artifact_config = config['upstream_deploy_artifact']
deploy_file_location = utils.ArtifactLocation(
artifact_config['pipeline_name'], artifact_config['stage_name'],
artifact_config['job_name'], artifact_config['artifact_name'])
# Create the armed stage as this pipeline needs to auto-execute
stages.generate_armed_stage(pipeline, constants.ARMED_JOB_NAME)
# Create a single stage in the pipeline which will rollback to the previous ASGs/AMI.
rollback_stage = stages.generate_rollback_asg_stage(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['hipchat_token'],
constants.HIPCHAT_ROOM,
deploy_file_location,
)
# Since we only want this stage to rollback via manual approval, ensure that it is set on this stage.
rollback_stage.set_has_manual_approval()
def generate_cleanup_stages(pipeline, config):
#
# Create the stage to terminate the EC2 instance used to both build the AMI and run DB migrations.
#
instance_info_location = utils.ArtifactLocation(
config['pipeline_name_build'], constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
stages.generate_terminate_instance(
pipeline,
instance_info_location,
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_auth_token=config['hipchat_token'],
runif='any')
return pipeline
def generate_deploy_stages(pipeline, config):
#
# Create the stage to deploy the AMI.
#
ami_file_location = utils.ArtifactLocation(config['pipeline_name_build'],
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME)
stages.generate_deploy_ami(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
ami_file_location,
manual_approval=not config.get('auto_deploy_ami', False))
return pipeline
def builder(pipeline, config):
"""
Add stages required to deploy edxapp to an environment to the
supplied pipeline.
Required Config Parameters:
asgard_api_endpoints
asgard_token
aws_access_key_id
aws_secret_access_key
github_token
edx_environment
"""
built_ami_file_location = utils.ArtifactLocation(
pipeline_name_build,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
)
stages.generate_deploy_ami(pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
built_ami_file_location,
manual_approval=not auto_deploy_ami)
pipeline.ensure_unencrypted_secure_environment_variables(
{'GITHUB_TOKEN': config['github_token']})
stages.generate_deployment_messages(
pipeline=pipeline,
ami_pairs=ami_pairs,
stage_deploy_pipeline=stage_deploy_pipeline,
release_status=constants.ReleaseStatus[config['edx_environment']],
confluence_user=config['jira_user'],
confluence_password=config['jira_password'],
base_ami_artifact=base_ami_artifact,
head_ami_artifact=head_ami_artifact,
message_tags=[
('edx', 'edx-platform', 'edxapp-from-pipeline'),
('edx', 'edx-platform-private', 'edx_platform'),
],
github_token=config['github_token'],
)
return pipeline
def generate_cleanup_stages(pipeline, config, launch_stage):
"""
Create the stage to terminate the EC2 instance used to both build the AMI and run DB migrations.
Required Config Parameters:
aws_access_key_id
aws_secret_access_key
hipchat_token
"""
instance_info_location = utils.ArtifactLocation(
pipeline.name, launch_stage.name, constants.LAUNCH_INSTANCE_JOB_NAME,
constants.LAUNCH_INSTANCE_FILENAME)
stages.generate_terminate_instance(
pipeline,
instance_info_location,
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_token=config['hipchat_token'],
runif='any')
return pipeline
def install_pipelines(configurator, config):
"""
Arguments:
configurator (GoCdConfigurator)
config (dict)
env_config (dict)
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
Optional variables:
- configuration_secure_version
- configuration_internal_version
"""
configurator.ensure_removal_of_pipeline_group('edxapp')
configurator.ensure_removal_of_pipeline_group('edxapp_prod_deploys')
edxapp_group = configurator.ensure_pipeline_group('edxapp')
ensure_permissions(configurator, edxapp_group, Permission.OPERATE, ['edxapp-operator'])
ensure_permissions(configurator, edxapp_group, Permission.VIEW, ['edxapp-operator'])
edxapp_deploy_group = configurator.ensure_pipeline_group('edxapp_prod_deploys')
ensure_permissions(configurator, edxapp_deploy_group, Permission.ADMINS, ['deploy'])
ensure_permissions(configurator, edxapp_deploy_group, Permission.OPERATE, ['prod-deploy-operators'])
ensure_permissions(configurator, edxapp_deploy_group, Permission.VIEW, ['prod-deploy-operators'])
cut_branch = edxapp.make_release_candidate(
edxapp_group,
config,
)
cut_branch.set_label_template('${edx-platform[:7]}')
prerelease_materials = edxapp.prerelease_materials(
edxapp_group,
config
)
prerelease_merge_artifact = utils.ArtifactLocation(
prerelease_materials.name,
constants.PRERELEASE_MATERIALS_STAGE_NAME,
constants.PRERELEASE_MATERIALS_JOB_NAME,
constants.PRIVATE_RC_FILENAME,
)
stage_b = edxapp.launch_and_terminate_subset_pipeline(
edxapp_group,
[
edxapp.generate_build_stages(
app_repo=EDX_PLATFORM().url,
edp=STAGE_EDX_EDXAPP,
theme_url=EDX_MICROSITE().url,
configuration_secure_repo=EDX_SECURE().url,
configuration_internal_repo=EDX_INTERNAL().url,
configuration_url=CONFIGURATION().url,
prerelease_merge_artifact=prerelease_merge_artifact,
),
],
config=config[edxapp.STAGE_EDX_EDXAPP],
pipeline_name="STAGE_edxapp_B",
ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(STAGE_EDX_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
auto_run=True,
)
stage_b.set_label_template('${prerelease}')
prod_edx_b = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.generate_build_stages(
app_repo=EDX_PLATFORM().url,
edp=PROD_EDX_EDXAPP,
theme_url=EDX_MICROSITE().url,
configuration_secure_repo=EDX_SECURE().url,
configuration_internal_repo=EDX_INTERNAL().url,
configuration_url=CONFIGURATION().url,
prerelease_merge_artifact=prerelease_merge_artifact,
),
],
config=config[edxapp.PROD_EDX_EDXAPP],
pipeline_name="PROD_edx_edxapp_B",
ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDX_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
auto_run=True,
)
prod_edx_b.set_label_template('${prerelease}')
prod_edge_b = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.generate_build_stages(
app_repo=EDX_PLATFORM().url,
edp=PROD_EDGE_EDXAPP,
theme_url=EDX_MICROSITE().url,
configuration_secure_repo=EDGE_SECURE().url,
configuration_internal_repo=EDGE_INTERNAL().url,
configuration_url=CONFIGURATION().url,
prerelease_merge_artifact=prerelease_merge_artifact,
),
],
config=config[edxapp.PROD_EDGE_EDXAPP],
pipeline_name="PROD_edge_edxapp_B",
ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDGE_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
auto_run=True,
)
prod_edge_b.set_label_template('${prerelease}')
for pipeline in (stage_b, prod_edx_b, prod_edge_b):
pipeline.ensure_material(
PipelineMaterial(
pipeline_name=prerelease_materials.name,
stage_name=constants.BASE_AMI_SELECTION_STAGE_NAME,
material_name="prerelease",
)
)
deployed_ami_pairs = [
(
utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
ami_selection_job_name,
constants.BASE_AMI_OVERRIDE_FILENAME,
),
utils.ArtifactLocation(
build_pipeline.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
)
) for build_pipeline, ami_selection_job_name in [
(prod_edx_b, constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDX_EDXAPP)),
(prod_edge_b, constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDGE_EDXAPP))
]
]
stage_md = edxapp.launch_and_terminate_subset_pipeline(
edxapp_group,
stage_builders=[
edxapp.generate_migrate_stages,
edxapp.generate_deploy_stages(
pipeline_name_build=stage_b.name,
ami_pairs=deployed_ami_pairs,
stage_deploy_pipeline=None,
base_ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(STAGE_EDX_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
head_ami_artifact=utils.ArtifactLocation(
stage_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
auto_deploy_ami=True,
),
],
post_cleanup_builders=[
edxapp.generate_e2e_test_stage,
],
config=config[edxapp.STAGE_EDX_EDXAPP],
pipeline_name="STAGE_edxapp_M-D",
ami_artifact=utils.ArtifactLocation(
stage_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
auto_run=True,
)
stage_md.set_automatic_pipeline_locking()
stage_md.set_label_template('${STAGE_edxapp_B_build}')
for build_stage in (stage_b, prod_edx_b, prod_edge_b):
stage_md.ensure_material(
PipelineMaterial(
pipeline_name=build_stage.name,
stage_name=constants.BUILD_AMI_STAGE_NAME,
material_name="{}_build".format(build_stage.name),
)
)
stage_md.ensure_material(
PipelineMaterial(
pipeline_name=prerelease_materials.name,
stage_name=constants.BASE_AMI_SELECTION_STAGE_NAME,
material_name="prerelease",
)
)
rollback_stage_db = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.rollback_database(edxapp.STAGE_EDX_EDXAPP, stage_b, stage_md),
],
config=config[edxapp.STAGE_EDX_EDXAPP],
pipeline_name="stage_edxapp_Rollback_Migrations",
ami_artifact=utils.ArtifactLocation(
stage_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME
),
auto_run=False,
pre_launch_builders=[
edxapp.armed_stage_builder,
],
)
rollback_stage_db.set_label_template('${deploy_pipeline}')
manual_verification = edxapp.manual_verification(
edxapp_deploy_group,
config
)
manual_verification.set_label_template('${stage_ami_deploy}')
manual_verification.ensure_material(
PipelineMaterial(
pipeline_name=stage_md.name,
stage_name=constants.TERMINATE_INSTANCE_STAGE_NAME,
material_name='stage_ami_deploy',
)
)
manual_verification.ensure_material(
PipelineMaterial(
pipeline_name=prod_edx_b.name,
stage_name=constants.BUILD_AMI_STAGE_NAME,
material_name='PROD_edx_edxapp_ami_build',
)
)
manual_verification.ensure_material(
PipelineMaterial(
pipeline_name=prod_edge_b.name,
stage_name=constants.BUILD_AMI_STAGE_NAME,
material_name='PROD_edge_edxapp_ami_build',
)
)
release_advancer = edxapp.release_advancer(
edxapp_deploy_group,
config
)
release_advancer.set_label_template('${tubular[:7]}-${COUNT}')
# When manually triggered in the pipeline above, the following two pipelines migrate/deploy
# to the production EDX and EDGE environments.
prod_edx_md = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.generate_migrate_stages,
edxapp.generate_deploy_stages(
pipeline_name_build=prod_edx_b.name,
ami_pairs=deployed_ami_pairs,
stage_deploy_pipeline=stage_md,
base_ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDX_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
head_ami_artifact=utils.ArtifactLocation(
prod_edx_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
auto_deploy_ami=True,
)
],
config=config[edxapp.PROD_EDX_EDXAPP],
pipeline_name="PROD_edx_edxapp_M-D",
ami_artifact=utils.ArtifactLocation(
prod_edx_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
auto_run=True,
)
prod_edx_md.set_label_template('${prod_release_gate}')
prod_edge_md = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.generate_migrate_stages,
edxapp.generate_deploy_stages(
pipeline_name_build=prod_edge_b.name,
ami_pairs=deployed_ami_pairs,
stage_deploy_pipeline=stage_md,
base_ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDGE_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
head_ami_artifact=utils.ArtifactLocation(
prod_edge_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
auto_deploy_ami=True,
)
],
config=config[edxapp.PROD_EDGE_EDXAPP],
pipeline_name="PROD_edge_edxapp_M-D",
ami_artifact=utils.ArtifactLocation(
prod_edge_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
auto_run=True,
)
prod_edge_md.set_label_template('${prod_release_gate}')
for deploy in (prod_edx_md, prod_edge_md):
deploy.ensure_material(
PipelineMaterial(
pipeline_name=manual_verification.name,
stage_name=constants.MANUAL_VERIFICATION_STAGE_NAME,
material_name="prod_release_gate",
)
)
for build in (prod_edx_b, prod_edge_b):
deploy.ensure_material(
PipelineMaterial(build.name, constants.BUILD_AMI_STAGE_NAME, "{}_build".format(build.name))
)
deploy.ensure_material(
PipelineMaterial(stage_md.name, constants.TERMINATE_INSTANCE_STAGE_NAME, "terminate_instance_stage")
)
deploy.ensure_material(
PipelineMaterial(
pipeline_name=prerelease_materials.name,
stage_name=constants.BASE_AMI_SELECTION_STAGE_NAME,
material_name="prerelease",
)
)
for pipeline in (stage_b, stage_md, prod_edx_b, prod_edx_md, prod_edge_b, prod_edge_md):
for material in (
TUBULAR, CONFIGURATION, EDX_PLATFORM, EDX_SECURE, EDGE_SECURE,
EDX_MICROSITE, EDX_INTERNAL, EDGE_INTERNAL
):
pipeline.ensure_material(material())
rollback_edx = edxapp.rollback_asgs(
edxapp_deploy_group=edxapp_deploy_group,
pipeline_name='PROD_edx_edxapp_Rollback_latest',
deploy_pipeline=prod_edx_md,
config=config[edxapp.PROD_EDX_EDXAPP],
ami_pairs=deployed_ami_pairs,
stage_deploy_pipeline=stage_md,
base_ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDX_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
head_ami_artifact=utils.ArtifactLocation(
prod_edx_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
)
rollback_edx.set_label_template('${deploy_ami}')
rollback_edge = edxapp.rollback_asgs(
edxapp_deploy_group=edxapp_deploy_group,
pipeline_name='PROD_edge_edxapp_Rollback_latest',
deploy_pipeline=prod_edge_md,
config=config[edxapp.PROD_EDGE_EDXAPP],
ami_pairs=deployed_ami_pairs,
stage_deploy_pipeline=stage_md,
base_ami_artifact=utils.ArtifactLocation(
prerelease_materials.name,
constants.BASE_AMI_SELECTION_STAGE_NAME,
constants.BASE_AMI_SELECTION_EDP_JOB_NAME(PROD_EDGE_EDXAPP),
constants.BASE_AMI_OVERRIDE_FILENAME,
),
head_ami_artifact=utils.ArtifactLocation(
prod_edge_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME,
),
)
rollback_edge.set_label_template('${deploy_ami}')
for rollback_pipeline in (rollback_edx, rollback_edge):
rollback_pipeline.ensure_material(
PipelineMaterial(
pipeline_name=stage_md.name,
stage_name=constants.TERMINATE_INSTANCE_STAGE_NAME,
material_name='terminate_instance_stage',
)
)
rollback_pipeline.ensure_material(
PipelineMaterial(
pipeline_name=prerelease_materials.name,
stage_name=constants.BASE_AMI_SELECTION_STAGE_NAME,
material_name="prerelease",
)
)
for build in (prod_edx_b, prod_edge_b):
rollback_pipeline.ensure_material(
PipelineMaterial(
pipeline_name=build.name,
stage_name=constants.BUILD_AMI_STAGE_NAME,
material_name='{}_build_ami'.format(build.name),
)
)
rollback_edx.ensure_material(
PipelineMaterial(prod_edx_md.name, constants.DEPLOY_AMI_STAGE_NAME, "deploy_ami")
)
rollback_edge.ensure_material(
PipelineMaterial(prod_edge_md.name, constants.DEPLOY_AMI_STAGE_NAME, "deploy_ami")
)
rollback_edx_db = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.rollback_database(edxapp.PROD_EDX_EDXAPP, prod_edx_b, prod_edx_md),
],
config=config[edxapp.PROD_EDX_EDXAPP],
pipeline_name="PROD_edx_edxapp_Rollback_Migrations_latest",
ami_artifact=utils.ArtifactLocation(
prod_edx_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME
),
auto_run=False,
pre_launch_builders=[
edxapp.armed_stage_builder,
],
)
rollback_edx_db.set_label_template('${deploy_pipeline}')
rollback_edge_db = edxapp.launch_and_terminate_subset_pipeline(
edxapp_deploy_group,
[
edxapp.rollback_database(edxapp.PROD_EDGE_EDXAPP, prod_edge_b, prod_edge_md),
],
config=config[edxapp.PROD_EDGE_EDXAPP],
pipeline_name="PROD_edge_edxapp_Rollback_Migrations_latest",
ami_artifact=utils.ArtifactLocation(
prod_edge_b.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
constants.BUILD_AMI_FILENAME
),
auto_run=False,
pre_launch_builders=[
edxapp.armed_stage_builder,
],
)
rollback_edge_db.set_label_template('${deploy_pipeline}')
deploy_artifact = utils.ArtifactLocation(
prod_edx_md.name,
constants.DEPLOY_AMI_STAGE_NAME,
constants.DEPLOY_AMI_JOB_NAME,
constants.DEPLOY_AMI_OUT_FILENAME,
)
merge_back = edxapp.merge_back_branches(
edxapp_deploy_group,
constants.BRANCH_CLEANUP_PIPELINE_NAME,
deploy_artifact,
prerelease_merge_artifact,
config,
)
merge_back.set_label_template('${{deploy_pipeline_{}}}'.format(prod_edx_md.name))
merge_back.ensure_material(
PipelineMaterial(
pipeline_name=prerelease_materials.name,
stage_name=constants.PRERELEASE_MATERIALS_STAGE_NAME,
material_name='prerelease_materials',
)
)
# Specify the upstream deploy pipeline materials for this branch-merging pipeline.
for deploy_pipeline in (prod_edx_md, prod_edge_md):
merge_back.ensure_material(
PipelineMaterial(
pipeline_name=deploy_pipeline.name,
stage_name=constants.DEPLOY_AMI_STAGE_NAME,
material_name='deploy_pipeline_{}'.format(deploy_pipeline.name),
)
)
def install_pipelines(configurator, config):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
Optional variables:
- configuration_secure_version
- configuration_internal_version
"""
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])
# Example materials yaml
# materials:
# - url: "https://github.com/edx/tubular"
# branch: "release"
# material_name: "tubular"
# polling: "True"
# destination_directory: "tubular"
# ignore_patterns:
# - '**/*'
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=set(material['ignore_patterns'])))
# If no upstream pipelines exist, don't install them!
for material in config.get('upstream_pipelines', []):
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
#
# Create the AMI-building stage.
#
stages.generate_launch_instance(
pipeline,
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['ec2_vpc_subnet_id'],
config['ec2_security_group_id'],
config['ec2_instance_profile_name'],
config['base_ami_id'],
manual_approval=not config.get('auto_run', False))
stages.generate_run_play(
pipeline,
'playbooks/edx-east/edxapp.yml',
edp=utils.EDP(config['edx_environment'], config['edx_deployment'],
config['play_name']),
private_github_key=config['github_private_key'],
app_repo=config['app_repo'],
configuration_secure_dir='{}-secure'.format(config['edx_deployment']),
configuration_internal_dir='{}-internal'.format(
config['edx_deployment']),
hipchat_token=config['hipchat_token'],
hipchat_room='release',
edx_platform_version='$GO_REVISION_EDX_PLATFORM',
edx_platform_repo='$APP_REPO',
configuration_version='$GO_REVISION_CONFIGURATION',
edxapp_theme_source_repo=config['theme_url'],
edxapp_theme_version='$GO_REVISION_EDX_THEME',
edxapp_theme_name='$EDXAPP_THEME_NAME',
disable_edx_services='true',
COMMON_TAG_EC2_INSTANCE='true',
cache_id='$GO_PIPELINE_COUNTER')
configuration_secure_repo = config['{}_configuration_secure_repo'.format(
config['edx_deployment'])]
configuration_internal_repo = config[
'{}_configuration_internal_repo'.format(config['edx_deployment'])]
configuration_secure_version = '$GO_REVISION_{}_SECURE'.format(
config['edx_deployment'].upper())
configuration_internal_version = '$GO_REVISION_{}_INTERNAL'.format(
config['edx_deployment'].upper())
stages.generate_create_ami_from_instance(
pipeline,
edp=utils.EDP(config['edx_environment'], config['edx_deployment'],
config['play_name']),
app_repo=config['app_repo'],
app_version='$GO_REVISION_EDX_PLATFORM',
hipchat_token=config['hipchat_token'],
hipchat_room='release pipeline',
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
version_tags={
'configuration':
(config['configuration_url'], '$GO_REVISION_CONFIGURATION'),
'configuration_secure':
(configuration_secure_repo, configuration_secure_version),
'configuration_internal':
(configuration_internal_repo, configuration_internal_version),
'edxapp_theme':
(config['theme_url'], '$GO_REVISION_EDX_MICROSITE'),
})
#
# Create the DB migration running stage.
#
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME,
constants.ANSIBLE_INVENTORY_FILENAME)
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.KEY_PEM_FILENAME)
launch_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
for sub_app in ['cms', 'lms']:
stages.generate_run_migrations(
pipeline,
db_migration_pass=config['db_migration_pass'],
inventory_location=ansible_inventory_location,
instance_key_location=instance_ssh_key_location,
launch_info_location=launch_info_location,
application_user=config['db_migration_user'],
application_name=config['play_name'],
application_path=config['application_path'],
sub_application_name=sub_app)
#
# Create the stage to deploy the AMI.
#
ami_file_location = utils.ArtifactLocation(pipeline.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
'ami.yml')
stages.generate_deploy_ami(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
ami_file_location,
manual_approval=not config.get('auto_deploy_ami', False))
#
# Create the stage to terminate the EC2 instance used to both build the AMI and run DB migrations.
#
instance_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME)
stages.generate_terminate_instance(
pipeline,
instance_info_location,
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_token=config['hipchat_token'],
runif='any')
def generate_basic_multistage_pipeline(play,
pipeline_group,
playbook_path,
app_repo,
service_name,
hipchat_room,
config,
save_config_locally,
dry_run,
post_migration_stages=(),
skip_migrations=False,
**kwargs):
"""
This pattern generates a pipeline that is suitable for the majority of edX's independently-deployable applications
(IDAs).
The generated pipeline will includes stages that do the following:
1. Launch a new instance on which we will build an AMI.
2. Run the Ansible play for the service.
3. Create an AMI based on the instance.
4. Run migrations.
5. Deploy the AMI (after manual intervention)
6. Destroy the instance on which the AMI was built.
Notes:
The instance launched/destroyed is NEVER inserted into the load balancer or serving user requests.
"""
environment = config['edx_environment']
deployment = config['edx_deployment']
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
application_name = service_name
application_path = '/edx/app/' + service_name
application_user = service_name
hipchat_token = config['hipchat_token']
pipeline = gcc.ensure_pipeline_group(pipeline_group) \
.ensure_replacement_of_pipeline('-'.join([environment, deployment, play])) \
.ensure_material(GitMaterial(config['tubular_url'],
branch=config.get('tubular_version', 'master'),
material_name='tubular',
polling=True,
destination_directory='tubular',
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX)) \
.ensure_material(GitMaterial(config['configuration_url'],
branch=config.get('configuration_version', 'master'),
material_name='configuration',
polling=True,
destination_directory='configuration',
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX)) \
.ensure_material(GitMaterial(config['app_repo'],
branch=config.get('app_version', 'master'),
material_name=play,
polling=True,
destination_directory=config['app_destination_directory'])) \
.ensure_material(GitMaterial(config['configuration_secure_repo'],
branch=config.get('configuration_secure_version', 'master'),
material_name='configuration_secure',
polling=True,
destination_directory=constants.PRIVATE_CONFIGURATION_LOCAL_DIR,
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX)) \
.ensure_material(GitMaterial(config['configuration_internal_repo'],
branch=config.get('configuration_internal_version', 'master'),
material_name='configuration_internal',
polling=True,
destination_directory=constants.INTERNAL_CONFIGURATION_LOCAL_DIR,
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX)) \
.ensure_environment_variables({
'APPLICATION_USER': application_user,
'APPLICATION_NAME': application_name,
'APPLICATION_PATH': application_path,
})
# Launch a new instance on which to build the AMI
stages.generate_launch_instance(
pipeline,
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['ec2_vpc_subnet_id'],
config['ec2_security_group_id'],
config['ec2_instance_profile_name'],
config['base_ami_id'],
manual_approval=not config.get('auto_run', False))
# Run the Ansible play for the service
stages.generate_run_play(
pipeline,
playbook_with_path=playbook_path,
play=play,
deployment=deployment,
edx_environment=environment,
app_repo=app_repo,
configuration_secure_repo=config['configuration_secure_repo'],
configuration_secure_dir=constants.PRIVATE_CONFIGURATION_LOCAL_DIR,
# remove above line and uncomment the below once materials are changed over to list.
# configuration_secure_dir='{}-secure'.format(config['edx_deployment']),
private_github_key=config['github_private_key'],
hipchat_auth_token=hipchat_token,
hipchat_room=hipchat_room,
disable_edx_services='true',
COMMON_TAG_EC2_INSTANCE='true',
**kwargs)
# Create an AMI
stages.generate_create_ami_from_instance(
pipeline,
play=play,
deployment=deployment,
edx_environment=environment,
app_repo=app_repo,
configuration_secure_repo=config['configuration_secure_repo'],
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_auth_token=hipchat_token,
hipchat_room=hipchat_room,
configuration_secure_version='$GO_REVISION_CONFIGURATION_SECURE',
# remove above line and uncomment the below once materials are changed over to list.
# configuration_secure_version='$GO_REVISION_{}_SECURE'.format(config['edx_deployment'].upper()),
**kwargs)
# Run database migrations
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'ansible_inventory')
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'key.pem')
launch_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'launch_info.yml')
if not skip_migrations:
stages.generate_run_migrations(pipeline,
config['db_migration_pass'],
ansible_inventory_location,
instance_ssh_key_location,
launch_info_location,
application_user=application_user,
application_name=application_name,
application_path=application_path)
# Run post-migration stages/tasks
for stage in post_migration_stages:
stage(pipeline,
ansible_inventory_location,
instance_ssh_key_location,
launch_info_location,
application_user=application_user,
application_name=application_name,
application_path=application_path,
hipchat_auth_token=hipchat_token,
hipchat_room=hipchat_room)
# Deploy the AMI (after user manually approves)
ami_file_location = utils.ArtifactLocation(pipeline.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
'ami.yml')
stages.generate_deploy_ami(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
ami_file_location,
manual_approval=not config.get('auto_deploy_ami', False))
# Terminate the instance used to create the AMI and run migrations. It was never inserted into the
# load balancer, or serving requests, so this is safe.
instance_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'launch_info.yml')
stages.generate_terminate_instance(
pipeline,
instance_info_location,
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_auth_token=hipchat_token,
runif='any')
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def builder(pipeline, config):
"""
Add database rollback stages to ``pipeline``.
"""
for material in (
TUBULAR,
CONFIGURATION,
EDX_PLATFORM,
EDX_SECURE,
EDGE_SECURE,
EDX_MICROSITE,
EDX_INTERNAL,
EDGE_INTERNAL,
):
pipeline.ensure_material(material())
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME,
constants.ANSIBLE_INVENTORY_FILENAME)
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, constants.KEY_PEM_FILENAME)
# Specify the upstream deploy pipeline material for this rollback pipeline.
# Assumes there's only a single upstream pipeline material for this pipeline.
pipeline.ensure_material(
PipelineMaterial(
pipeline_name=deploy_pipeline.name,
stage_name=constants.DEPLOY_AMI_STAGE_NAME,
material_name='deploy_pipeline',
))
# We need the build_pipeline upstream so that we can fetch the AMI selection artifact from it
pipeline.ensure_material(
PipelineMaterial(
pipeline_name=build_pipeline.name,
stage_name=constants.BUILD_AMI_STAGE_NAME,
material_name='select_base_ami',
))
# Create a a stage for migration rollback.
for sub_app in EDXAPP_SUBAPPS:
migration_artifact = utils.ArtifactLocation(
deploy_pipeline.name,
constants.APPLY_MIGRATIONS_STAGE + "_" + sub_app,
constants.APPLY_MIGRATIONS_JOB,
constants.MIGRATION_OUTPUT_DIR_NAME,
is_dir=True)
stages.generate_rollback_migrations(
pipeline,
edp,
db_migration_pass=config['db_migration_pass'],
inventory_location=ansible_inventory_location,
instance_key_location=instance_ssh_key_location,
migration_info_location=migration_artifact,
application_user=config['db_migration_user'],
application_name=config['play_name'],
application_path=config['application_path'],
sub_application_name=sub_app)
return pipeline
def rollback_asgs(
edxapp_deploy_group,
pipeline_name,
deploy_pipeline,
config,
ami_pairs,
stage_deploy_pipeline,
base_ami_artifact,
head_ami_artifact,
):
"""
Arguments:
edxapp_deploy_group (gomatic.PipelineGroup): The group in which to create this pipeline
pipeline_name (str): The name of this pipeline
deploy_pipeline (gomatic.Pipeline): The pipeline to retrieve the ami_deploy_info.yml artifact from
config (dict): the configuraiton dictionary
ami_pairs (list<tuple>): A list of tuples. The first item in the tuple should be Artifact location of the
base_ami ID that was running before deployment and the ArtifactLocation of the newly deployed AMI ID
e.g. (ArtifactLocation
(pipeline='prerelease_edxapp_materials_latest',
stage='select_base_ami', job='select_base_ami_prod_edx_job',
file_name='ami_override.yml',
is_dir=False
),
ArtifactLocation
(pipeline='PROD_edx_edxapp_B',
stage='build_ami',
job='build_ami_job',
file_name='ami.yml',
is_dir=False
)
)
stage_deploy_pipeline (gomatic.Pipeline): The edxapp staging deployment pipeline
base_ami_artifact (edxpipelines.utils.ArtifactLocation): ArtifactLocation of the base AMI selection
head_ami_artifact (edxpipelines.utils.ArtifactLocation): ArtifactLocation of the head AMI selection
Configuration Required:
tubular_sleep_wait_time
asgard_api_endpoints
asgard_token
aws_access_key_id
aws_secret_access_key
hipchat_token
"""
pipeline = edxapp_deploy_group.ensure_replacement_of_pipeline(pipeline_name)\
.ensure_environment_variables({'WAIT_SLEEP_TIME': config['tubular_sleep_wait_time']})
for material in (
TUBULAR,
CONFIGURATION,
EDX_PLATFORM,
EDX_SECURE,
EDGE_SECURE,
EDX_MICROSITE,
EDX_INTERNAL,
EDGE_INTERNAL,
):
pipeline.ensure_material(material())
# Specify the artifact that will be fetched containing the previous deployment information.
deploy_file_location = utils.ArtifactLocation(
deploy_pipeline.name,
constants.DEPLOY_AMI_STAGE_NAME,
constants.DEPLOY_AMI_JOB_NAME,
constants.DEPLOY_AMI_OUT_FILENAME,
)
# Create the armed stage as this pipeline needs to auto-execute
stages.generate_armed_stage(pipeline, constants.ARMED_JOB_NAME)
# Create a single stage in the pipeline which will rollback to the previous ASGs/AMI.
rollback_stage = stages.generate_rollback_asg_stage(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['hipchat_token'],
constants.HIPCHAT_ROOM,
deploy_file_location,
)
# Since we only want this stage to rollback via manual approval, ensure that it is set on this stage.
rollback_stage.set_has_manual_approval()
# Message PRs being rolled back
pipeline.ensure_unencrypted_secure_environment_variables(
{'GITHUB_TOKEN': config['github_token']})
stages.generate_deployment_messages(
pipeline=pipeline,
ami_pairs=ami_pairs,
stage_deploy_pipeline=stage_deploy_pipeline,
base_ami_artifact=base_ami_artifact,
head_ami_artifact=head_ami_artifact,
message_tags=[('edx', 'edx-platform', 'edxapp-from-pipeline'),
('edx', 'edx-platform-private', 'edx_platform')],
release_status=constants.ReleaseStatus.ROLLED_BACK,
confluence_user=config['jira_user'],
confluence_password=config['jira_password'],
github_token=config['github_token'],
)
return pipeline