def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
materials: A list of dictionaries of the materials used in this pipeline
upstream_pipelines: a list of dictionaries of the upstream pipelines that feed in to the manual verification
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = gcc.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=material['ignore_patterns']))
for material in config['upstream_pipelines']:
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
# What this accomplishes:
# When a pipeline such as edx stage runs this pipeline is downstream. Since the first stage is automatic
# the git materials will be carried over from the first pipeline.
#
# The second stage in this pipeline requires manual approval.
#
# This allows the overall workflow to remain paused while manual verification is completed and allows the git
# materials to stay pinned.
#
# Once the second phase is approved, the workflow will continue and pipelines downstream will continue to execute
# with the same pinned materials from the upstream pipeline.
stages.generate_armed_stage(pipeline,
constants.INITIAL_VERIFICATION_STAGE_NAME)
manual_verification_stage = pipeline.ensure_stage(
constants.MANUAL_VERIFICATION_STAGE_NAME)
manual_verification_stage.set_has_manual_approval()
manual_verification_job = manual_verification_stage.ensure_job(
constants.MANUAL_VERIFICATION_JOB_NAME)
manual_verification_job.add_task(
ExecTask([
'/bin/bash', '-c',
'echo Manual Verification run number $GO_PIPELINE_COUNTER completed by $GO_TRIGGER_USER'
], ))
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
pipelines.generate_basic_multistage_pipeline(
play='insights',
playbook_path='playbooks/edx-east/insights.yml',
app_repo='https://github.com/edx/edx-analytics-dashboard.git',
service_name='insights',
hipchat_room='Analytics',
pipeline_group='Analytics',
config=config,
dry_run=dry_run,
save_config_locally=save_config_locally,
app_version='$GO_REVISION_INSIGHTS',
INSIGHTS_VERSION='$GO_REVISION_INSIGHTS')
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
version_env_var = '$GO_REVISION_ECOMMERCE'
pipelines.generate_basic_multistage_pipeline(
play='ecommerce',
pipeline_group='E-Commerce',
playbook_path='playbooks/edx-east/ecommerce.yml',
app_repo='https://github.com/edx/ecommerce.git',
service_name='ecommerce',
hipchat_room='release',
config=config,
save_config_locally=save_config_locally,
dry_run=dry_run,
app_version=version_env_var,
ECOMMERCE_VERSION=version_env_var)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- pipeline_group
- pipeline_name
- asgard_api_endpoints
- asgard_token
- aws_access_key_id
- aws_secret_access_key
- cron_timer
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
configurator = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = configurator.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])\
.set_timer(config['cron_timer'])\
.set_git_material(GitMaterial("https://github.com/edx/tubular.git",
polling=True,
destination_directory="tubular",
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX
)
)
stages.generate_asg_cleanup(pipeline, config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'])
configurator.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- pipeline_name
- pipeline_group
- asgard_api_endpoints
- asgard_token
- aws_access_key_id
- aws_secret_access_key
To run this script:
python edxpipelines/pipelines/deploy_ami.py --variable_file ../gocd-pipelines/gocd/vars/tools/deploy_edge_ami.yml --variable_file ../gocd-pipelines/gocd/vars/tools/tools.yml
python edxpipelines/pipelines/deploy_ami.py --variable_file ../gocd-pipelines/gocd/vars/tools/deploy_edx_ami.yml --variable_file ../gocd-pipelines/gocd/vars/tools/tools.yml
python edxpipelines/pipelines/deploy_ami.py --variable_file ../gocd-pipelines/gocd/vars/tools/deploy-mckinsey-ami.yml --variable_file ../gocd-pipelines/gocd/vars/tools/tools.yml
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
configurator = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline_params = {
"pipeline_name": config['pipeline_name'],
"pipeline_group": config['pipeline_group'],
"asgard_api_endpoints": config['asgard_api_endpoints'],
"asgard_token": config['asgard_token'],
"aws_access_key_id": config['aws_access_key_id'],
"aws_secret_access_key": config['aws_secret_access_key']
}
configurator = pipelines.generate_deploy_pipeline(configurator,
**pipeline_params)
configurator.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
print "done"
def install_pipelines(save_config_locally, dry_run, bmd_steps, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
Optional variables:
- configuration_secure_version
- configuration_internal_version
"""
BMD_STAGES = {
'b': generate_build_stages,
'm': generate_migrate_stages,
'd': generate_deploy_stages
}
# Sort the BMD steps by the custom 'bmd' alphabet
bmd_steps = utils.sort_bmd(bmd_steps.lower())
# validate the caller has requested a valid pipeline configuration
utils.validate_pipeline_permutations(bmd_steps)
# Merge the configuration files/variables together
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
# Create the pipeline
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline_group = config['pipeline_group']
# Some pipelines will need to know the name of the upstream pipeline that built the AMI.
# Determine the build pipeline name and add it to the config.
pipeline_name, pipeline_name_build = utils.determine_pipeline_names(
config, bmd_steps)
if 'pipeline_name_build' in config:
raise Exception(
"The config 'pipeline_name_build' value exists but should only be programmatically generated!"
)
config['pipeline_name_build'] = pipeline_name_build
pipeline = gcc.ensure_pipeline_group(pipeline_group)\
.ensure_replacement_of_pipeline(pipeline_name)
# Setup the materials
# Example materials yaml
# materials:
# - url: "https://github.com/edx/tubular"
# branch: "master"
# material_name: "tubular"
# polling: "True"
# destination_directory: "tubular"
# ignore_patterns:
# - '**/*'
for material in config.get('materials', []):
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=material['ignore_patterns']))
# Setup the upstream pipeline materials
for material in config.get('upstream_pipelines', []):
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
# We always need to launch the AMI, independent deploys are done with a different pipeline
launch_stage = stages.generate_launch_instance(
pipeline,
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['ec2_vpc_subnet_id'],
config['ec2_security_group_id'],
config['ec2_instance_profile_name'],
config['base_ami_id'],
manual_approval=not config.get('auto_run', False))
# Generate all the requested stages
for phase in bmd_steps:
BMD_STAGES[phase](pipeline, config)
# Add the cleanup stage
generate_cleanup_stages(pipeline, config)
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
configurator = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = configurator \
.ensure_pipeline_group(DRUPAL_PIPELINE_GROUP_NAME) \
.ensure_replacement_of_pipeline('rollback-prod-marketing-site') \
.set_git_material(GitMaterial('https://github.com/edx/tubular',
polling=False,
destination_directory='tubular',
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX
)
) \
.ensure_material(PipelineMaterial(DEPLOY_MARKETING_PIPELINE_NAME, FETCH_TAG_STAGE_NAME))
pipeline.ensure_environment_variables({
'MARKETING_REPOSITORY_VERSION':
config['mktg_repository_version'],
})
pipeline.ensure_encrypted_environment_variables({
'PRIVATE_GITHUB_KEY':
config['github_private_key'],
'PRIVATE_MARKETING_REPOSITORY_URL':
config['mktg_repository_url'],
'PRIVATE_ACQUIA_USERNAME':
config['acquia_username'],
'PRIVATE_ACQUIA_PASSWORD':
config['acquia_password'],
'PRIVATE_ACQUIA_GITHUB_KEY':
config['acquia_github_key'],
})
prod_tag_name_artifact_params = {
'pipeline': DEPLOY_MARKETING_PIPELINE_NAME,
'stage': FETCH_TAG_STAGE_NAME,
'job': FETCH_TAG_JOB_NAME,
'src':
FetchArtifactFile('{prod_tag}.txt'.format(prod_tag=PROD_TAG_NAME)),
'dest': 'target'
}
# Stage to rollback stage to its last stable tag
rollback_stage = pipeline.ensure_stage(ROLLBACK_STAGE_NAME)
rollback_stage.set_has_manual_approval()
rollback_job = rollback_stage.ensure_job(ROLLBACK_JOB_NAME)
tasks.generate_requirements_install(rollback_job, 'tubular')
tasks.generate_target_directory(rollback_job)
rollback_job.add_task(FetchArtifactTask(**prod_tag_name_artifact_params))
tasks.generate_drupal_deploy(
rollback_job, PROD_ENV,
'{prod_tag}.txt'.format(prod_tag=PROD_TAG_NAME))
# Stage to clear caches in extra
clear_prod_caches_stage = pipeline.ensure_stage(
CLEAR_PROD_CACHES_STAGE_NAME)
clear_prod_caches_job = clear_prod_caches_stage.ensure_job(
CLEAR_PROD_CACHES_JOB_NAME)
tasks.fetch_edx_mktg(clear_prod_caches_job, 'edx-mktg')
tasks.generate_requirements_install(clear_prod_caches_job, 'tubular')
tasks.format_RSA_key(clear_prod_caches_job,
'edx-mktg/docroot/acquia_github_key.pem',
'$PRIVATE_ACQUIA_GITHUB_KEY')
tasks.generate_flush_drupal_caches(clear_prod_caches_job, PROD_ENV)
tasks.generate_clear_varnish_cache(clear_prod_caches_job, PROD_ENV)
configurator.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
artifact_path = 'target/'
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = gcc.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])\
.ensure_material(GitMaterial('https://github.com/edx/edx-gomatic',
material_name='edx-gomatic',
polling=True,
destination_directory='edx-gomatic',
branch='master'
)
) \
.ensure_material(GitMaterial('[email protected]:edx-ops/gomatic-secure.git',
material_name='gomatic-secure',
polling=True,
destination_directory='gomatic-secure',
branch='master',
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX
)
)
pipeline.ensure_encrypted_environment_variables({
'GOMATIC_USER':
config['gomatic_user'],
'GOMATIC_PASSWORD':
config['gomatic_password']
})
stage = pipeline.ensure_stage('deploy_gomatic_stage')
job = stage.ensure_job('deploy_gomatic_scripts_job')
tasks.generate_requirements_install(job, 'edx-gomatic')
job.add_task(
ExecTask([
'/usr/bin/python', 'deploy_pipelines.py', '-v', 'tools', '-f',
'config.yml'
],
working_dir='edx-gomatic'))
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
configurator = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = configurator \
.ensure_pipeline_group(config['pipeline']['group']) \
.ensure_replacement_of_pipeline(config['pipeline']['name']) \
.set_label_template('${api-manager}') \
.set_git_material(GitMaterial(config['github']['server_uri'] + '/' + config['github']['repository'],
branch='#{GIT_BRANCH}',
material_name='api-manager',
destination_directory=API_MANAGER_WORKING_DIR)
) \
pipeline.ensure_parameters({'GIT_BRANCH': config['github']['branch']})
pipeline.ensure_environment_variables({
'SWAGGER_CODEGEN_JAR':
config['swagger_codegen_jar'],
'GITHUB_API_REPO':
config['github']['repository'],
'GITHUB_API_URI':
config['github']['api_uri'],
'GITHUB_API_POLL_WAIT_S':
config['github']['api_poll_wait_s'],
'GITHUB_API_POLL_RETRIES':
config['github']['api_poll_retries']
})
# Note, need to move this Github poll hack to something less of a hack at some point.
setup_stage = pipeline.ensure_stage(SETUP_STAGE_NAME)
wait_for_travis_job = setup_stage.ensure_job(WAIT_FOR_TRAVIS_JOB_NAME)
wait_for_travis_job.add_task(
ExecTask([
'/bin/bash', '-c',
'i=0; until python -c "import requests; assert(requests.get(\'${GITHUB_API_URI}/${GITHUB_API_REPO}/commits/{}/status\'.format(\'${GO_REVISION_API_MANAGER}\')).json()[\'state\'] == \'success\')"; do i=$((i+1)); if [ $i -gt ${GITHUB_API_POLL_RETRIES} ]; then exit 1; fi; sleep ${GITHUB_API_POLL_WAIT_S}; done'
]))
download_stage = pipeline.ensure_stage(
DOWNLOAD_STAGE_NAME).set_clean_working_dir()
swagger_codegen_job = download_stage.ensure_job(
SWAGGER_CODEGEN_JOB_NAME).ensure_artifacts(
{BuildArtifact(SWAGGER_JAR)})
swagger_codegen_job.add_task(
ExecTask([
'/bin/bash', '-c',
'wget ${{SWAGGER_CODEGEN_JAR}} -O {swagger_jar}'.format(
swagger_jar=SWAGGER_JAR)
]))
build_stage = pipeline.ensure_stage(
BUILD_STAGE_NAME).set_clean_working_dir()
swagger_flatten_job = build_stage.ensure_job(
SWAGGER_FLATTEN_JOB_NAME).ensure_artifacts({
BuildArtifact('api-manager/swagger-build-artifacts/swagger.json')
})
artifact_params = {
'pipeline': pipeline.name,
'stage': DOWNLOAD_STAGE_NAME,
'job': SWAGGER_CODEGEN_JOB_NAME,
'src': FetchArtifactFile(SWAGGER_JAR),
'dest': API_MANAGER_WORKING_DIR
}
swagger_flatten_job.add_task(FetchArtifactTask(**artifact_params))
swagger_flatten_job.add_task(
ExecTask(['make', 'build'], working_dir=API_MANAGER_WORKING_DIR))
package_source_job = build_stage.ensure_job(
PACKAGE_SOURCE_JOB_NAME).ensure_artifacts(
{BuildArtifact('api-manager')})
package_source_job.add_task(
ExecTask([
'/bin/bash', '-c',
'/usr/bin/pip install -t python-libs -r requirements/base.txt'
],
working_dir=API_MANAGER_WORKING_DIR))
configurator.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def install_pipelines(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
Optional variables:
- configuration_secure_version
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = gcc.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])
# Example materials yaml
# materials:
# - url: "https://github.com/edx/tubular"
# branch: "master"
# material_name: "tubular"
# polling: "True"
# destination_directory: "tubular"
# ignore_patterns:
# - '**/*'
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=material['ignore_patterns']))
# If no upstream pipelines exist, don't install them!
for material in config.get('upstream_pipelines', []):
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
stages.generate_armed_stage(pipeline,
constants.PRERELEASE_MATERIALS_STAGE_NAME)
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def test_merge_files_and_dicts(self, expected, file_paths, *dicts):
merged = util.merge_files_and_dicts(file_paths, list(dicts))
print(merged)
self.assertEqual(merged, expected)
def install_pipeline(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
materials: List of dictionaries of the materials used in this pipeline
upstream_pipelines: List of dictionaries of the upstream piplines that feed in to the rollback pipeline.
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = gcc.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])\
.ensure_environment_variables({'WAIT_SLEEP_TIME': config['tubular_sleep_wait_time']})
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=material['ignore_patterns']))
# Specify the upstream deploy pipeline material for this rollback pipeline.
# Assumes there's only a single upstream pipeline material for this pipeline.
rollback_material = config['upstream_pipeline']
pipeline.ensure_material(
PipelineMaterial(pipeline_name=rollback_material['pipeline_name'],
stage_name=rollback_material['stage_name'],
material_name=rollback_material['material_name']))
# Specify the artifact that will be fetched containing the previous deployment information.
# Assumes there's only a single upstream artifact used by this pipeline.
artifact_config = config['upstream_deploy_artifact']
deploy_file_location = utils.ArtifactLocation(
artifact_config['pipeline_name'], artifact_config['stage_name'],
artifact_config['job_name'], artifact_config['artifact_name'])
# Create the armed stage as this pipeline needs to auto-execute
stages.generate_armed_stage(pipeline, constants.ARMED_JOB_NAME)
# Create a single stage in the pipeline which will rollback to the previous ASGs/AMI.
rollback_stage = stages.generate_rollback_asg_stage(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['hipchat_token'],
constants.HIPCHAT_ROOM,
deploy_file_location,
)
# Since we only want this stage to rollback via manual approval, ensure that it is set on this stage.
rollback_stage.set_has_manual_approval()
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)
def test_merge_files_and_dicts(self, expected, file_paths, *dicts):
merged = util.merge_files_and_dicts(file_paths, list(dicts))
print merged
self.assertEqual(merged, expected)
def install_pipeline(save_config_locally, dry_run, variable_files, cmd_line_vars):
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
configurator = GoCdConfigurator(
HostRestClient(config['gocd_url'], config['gocd_username'], config['gocd_password'], ssl=True))
pipeline = configurator \
.ensure_pipeline_group(DRUPAL_PIPELINE_GROUP_NAME) \
.ensure_replacement_of_pipeline(DEPLOY_MARKETING_PIPELINE_NAME) \
.set_git_material(GitMaterial('https://github.com/edx/tubular',
polling=True,
destination_directory='tubular',
ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX
)
)
pipeline.ensure_environment_variables(
{
'MARKETING_REPOSITORY_VERSION': config['mktg_repository_version'],
}
)
pipeline.ensure_encrypted_environment_variables(
{
'PRIVATE_GITHUB_KEY': config['github_private_key'],
'PRIVATE_MARKETING_REPOSITORY_URL': config['mktg_repository_url'],
'PRIVATE_ACQUIA_REMOTE': config['acquia_remote_url'],
'PRIVATE_ACQUIA_USERNAME': config['acquia_username'],
'PRIVATE_ACQUIA_PASSWORD': config['acquia_password'],
'PRIVATE_ACQUIA_GITHUB_KEY': config['acquia_github_key']
}
)
# Stage to fetch the current tag names from stage and prod
fetch_tag_stage = pipeline.ensure_stage(FETCH_TAG_STAGE_NAME)
fetch_tag_stage.set_has_manual_approval()
fetch_tag_job = fetch_tag_stage.ensure_job(FETCH_TAG_JOB_NAME)
tasks.generate_requirements_install(fetch_tag_job, 'tubular')
tasks.generate_target_directory(fetch_tag_job)
path_name = '../target/{env}_tag_name.txt'
tasks.generate_fetch_tag(fetch_tag_job, STAGE_ENV, path_name)
tasks.generate_fetch_tag(fetch_tag_job, PROD_ENV, path_name)
fetch_tag_job.ensure_artifacts(
set([BuildArtifact('target/{stage_tag}.txt'.format(stage_tag=STAGE_TAG_NAME)),
BuildArtifact('target/{prod_tag}.txt'.format(prod_tag=PROD_TAG_NAME))])
)
# Stage to create and push a tag to Acquia.
push_to_acquia_stage = pipeline.ensure_stage(PUSH_TO_ACQUIA_STAGE_NAME)
push_to_acquia_job = push_to_acquia_stage.ensure_job(PUSH_TO_ACQUIA_JOB_NAME)
# Ensures the tag name is accessible in future jobs.
push_to_acquia_job.ensure_artifacts(
set([BuildArtifact('target/{new_tag}.txt'.format(new_tag=NEW_TAG_NAME))])
)
tasks.generate_requirements_install(push_to_acquia_job, 'tubular')
tasks.generate_target_directory(push_to_acquia_job)
tasks.fetch_edx_mktg(push_to_acquia_job, 'edx-mktg')
# Create a tag from MARKETING_REPOSITORY_VERSION branch of marketing repo
push_to_acquia_job.add_task(
ExecTask(
[
'/bin/bash',
'-c',
# Writing dates to a file should help with any issues dealing with a job
# taking place over two days (23:59:59 -> 00:00:00). Only the day can be
# affected since we don't use minutes or seconds.
# NOTE: Uses UTC
'echo -n "release-$(date +%Y-%m-%d-%H.%M)" > ../target/{new_tag}.txt && '
'TAG_NAME=$(cat ../target/{new_tag}.txt) && '
'/usr/bin/git config user.email "*****@*****.**" && '
'/usr/bin/git config user.name "edx-secure" && '
'/usr/bin/git tag -a $TAG_NAME -m "Release for $(date +%B\ %d,\ %Y). Created by $GO_TRIGGER_USER." && '
'GIT_SSH_COMMAND="/usr/bin/ssh -o StrictHostKeyChecking=no -i ../github_key.pem" '
'/usr/bin/git push origin $TAG_NAME'.format(new_tag=NEW_TAG_NAME)
],
working_dir='edx-mktg'
)
)
# Set up Acquia Github key for use in pushing tag to Acquia
tasks.format_RSA_key(push_to_acquia_job, 'acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY')
# Set up Acquia remote repo and push tag to Acquia. Change new tag file to contain "tags/" for deployment.
push_to_acquia_job.add_task(
ExecTask(
[
'/bin/bash',
'-c',
'/usr/bin/git remote add acquia $PRIVATE_ACQUIA_REMOTE && '
'GIT_SSH_COMMAND="/usr/bin/ssh -o StrictHostKeyChecking=no -i ../acquia_github_key.pem" '
'/usr/bin/git push acquia $(cat ../target/{new_tag}.txt) && '
'echo -n "tags/" | cat - ../target/{new_tag}.txt > temp && mv temp ../target/{new_tag}.txt'.format(new_tag=NEW_TAG_NAME)
],
working_dir='edx-mktg'
)
)
# Stage to backup database in stage
backup_stage_database_stage = pipeline.ensure_stage(BACKUP_STAGE_DATABASE_STAGE_NAME)
backup_stage_database_job = backup_stage_database_stage.ensure_job(BACKUP_STAGE_DATABASE_JOB_NAME)
tasks.generate_requirements_install(backup_stage_database_job, 'tubular')
tasks.generate_backup_drupal_database(backup_stage_database_job, STAGE_ENV)
# Stage to deploy to stage
deploy_stage_for_stage = pipeline.ensure_stage(DEPLOY_STAGE_STAGE_NAME)
deploy_job_for_stage = deploy_stage_for_stage.ensure_job(DEPLOY_STAGE_JOB_NAME)
tasks.generate_requirements_install(deploy_job_for_stage, 'tubular')
tasks.generate_target_directory(deploy_job_for_stage)
# fetch the tag name
new_tag_name_artifact_params = {
'pipeline': DEPLOY_MARKETING_PIPELINE_NAME,
'stage': PUSH_TO_ACQUIA_STAGE_NAME,
'job': PUSH_TO_ACQUIA_JOB_NAME,
'src': FetchArtifactFile('{new_tag}.txt'.format(new_tag=NEW_TAG_NAME)),
'dest': 'target'
}
deploy_job_for_stage.add_task(FetchArtifactTask(**new_tag_name_artifact_params))
tasks.generate_drupal_deploy(deploy_job_for_stage, STAGE_ENV, '{new_tag}.txt'.format(new_tag=NEW_TAG_NAME))
# Stage to clear caches in stage
clear_stage_caches_stage = pipeline.ensure_stage(CLEAR_STAGE_CACHES_STAGE_NAME)
clear_stage_caches_job = clear_stage_caches_stage.ensure_job(CLEAR_STAGE_CACHES_JOB_NAME)
tasks.fetch_edx_mktg(clear_stage_caches_job, 'edx-mktg')
tasks.generate_requirements_install(clear_stage_caches_job, 'tubular')
tasks.format_RSA_key(clear_stage_caches_job, 'edx-mktg/docroot/acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY')
tasks.generate_flush_drupal_caches(clear_stage_caches_job, STAGE_ENV)
tasks.generate_clear_varnish_cache(clear_stage_caches_job, STAGE_ENV)
# Stage to backup database in prod
backup_prod_database_stage = pipeline.ensure_stage(BACKUP_PROD_DATABASE_STAGE_NAME)
backup_prod_database_stage.set_has_manual_approval()
backup_prod_database_job = backup_prod_database_stage.ensure_job(BACKUP_PROD_DATABASE_JOB_NAME)
tasks.generate_requirements_install(backup_prod_database_job, 'tubular')
tasks.generate_backup_drupal_database(backup_prod_database_job, PROD_ENV)
# Stage to deploy to prod
deploy_stage_for_prod = pipeline.ensure_stage(DEPLOY_PROD_STAGE_NAME)
deploy_job_for_prod = deploy_stage_for_prod.ensure_job(DEPLOY_PROD_JOB_NAME)
tasks.generate_requirements_install(deploy_job_for_prod, 'tubular')
tasks.generate_target_directory(deploy_job_for_prod)
deploy_job_for_prod.add_task(FetchArtifactTask(**new_tag_name_artifact_params))
tasks.generate_drupal_deploy(deploy_job_for_prod, PROD_ENV, '{new_tag}.txt'.format(new_tag=NEW_TAG_NAME))
# Stage to clear caches in prod
clear_prod_caches_stage = pipeline.ensure_stage(CLEAR_PROD_CACHES_STAGE_NAME)
clear_prod_caches_job = clear_prod_caches_stage.ensure_job(CLEAR_PROD_CACHES_JOB_NAME)
tasks.fetch_edx_mktg(clear_prod_caches_job, 'edx-mktg')
tasks.generate_requirements_install(clear_prod_caches_job, 'tubular')
tasks.format_RSA_key(clear_prod_caches_job, 'edx-mktg/docroot/acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY')
tasks.generate_flush_drupal_caches(clear_prod_caches_job, PROD_ENV)
tasks.generate_clear_varnish_cache(clear_prod_caches_job, PROD_ENV)
configurator.save_updated_config(save_config_locally=save_config_locally, dry_run=dry_run)
def install_pipelines(save_config_locally, dry_run, variable_files,
cmd_line_vars):
"""
Variables needed for this pipeline:
- gocd_username
- gocd_password
- gocd_url
- configuration_secure_repo
- configuration_internal_repo
- hipchat_token
- github_private_key
- aws_access_key_id
- aws_secret_access_key
- ec2_vpc_subnet_id
- ec2_security_group_id
- ec2_instance_profile_name
- base_ami_id
Optional variables:
- configuration_secure_version
- configuration_internal_version
"""
config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, ))
gcc = GoCdConfigurator(
HostRestClient(config['gocd_url'],
config['gocd_username'],
config['gocd_password'],
ssl=True))
pipeline = gcc.ensure_pipeline_group(config['pipeline_group'])\
.ensure_replacement_of_pipeline(config['pipeline_name'])
# Example materials yaml
# materials:
# - url: "https://github.com/edx/tubular"
# branch: "master"
# material_name: "tubular"
# polling: "True"
# destination_directory: "tubular"
# ignore_patterns:
# - '**/*'
for material in config['materials']:
pipeline.ensure_material(
GitMaterial(
url=material['url'],
branch=material['branch'],
material_name=material['material_name'],
polling=material['polling'],
destination_directory=material['destination_directory'],
ignore_patterns=material['ignore_patterns']))
# If no upstream pipelines exist, don't install them!
for material in config.get('upstream_pipelines', []):
pipeline.ensure_material(
PipelineMaterial(pipeline_name=material['pipeline_name'],
stage_name=material['stage_name'],
material_name=material['material_name']))
#
# Create the AMI-building stage.
#
launch_stage = stages.generate_launch_instance(
pipeline,
config['aws_access_key_id'],
config['aws_secret_access_key'],
config['ec2_vpc_subnet_id'],
config['ec2_security_group_id'],
config['ec2_instance_profile_name'],
config['base_ami_id'],
manual_approval=not config.get('auto_run', False))
stages.generate_run_play(
pipeline,
'playbooks/edx-east/edxapp.yml',
play=config['play_name'],
deployment=config['edx_deployment'],
edx_environment=config['edx_environment'],
private_github_key=config['github_private_key'],
app_repo=config['app_repo'],
configuration_secure_dir='{}-secure'.format(config['edx_deployment']),
configuration_internal_dir='{}-internal'.format(
config['edx_deployment']),
hipchat_token=config['hipchat_token'],
hipchat_room='release',
edx_platform_version='$GO_REVISION_EDX_PLATFORM',
edx_platform_repo='$APP_REPO',
configuration_version='$GO_REVISION_CONFIGURATION',
edxapp_theme_source_repo=config['theme_url'],
edxapp_theme_version='$GO_REVISION_EDX_THEME',
edxapp_theme_name='$EDXAPP_THEME_NAME',
disable_edx_services='true',
COMMON_TAG_EC2_INSTANCE='true',
cache_id='$GO_PIPELINE_COUNTER')
stages.generate_create_ami_from_instance(
pipeline,
play=config['play_name'],
deployment=config['edx_deployment'],
edx_environment=config['edx_environment'],
app_repo=config['app_repo'],
app_version='$GO_REVISION_EDX_PLATFORM',
configuration_secure_repo=config['{}_configuration_secure_repo'.format(
config['edx_deployment'])],
configuration_internal_repo=config[
'{}_configuration_internal_repo'.format(config['edx_deployment'])],
configuration_repo=config['configuration_url'],
hipchat_auth_token=config['hipchat_token'],
hipchat_room='release pipeline',
configuration_version='$GO_REVISION_CONFIGURATION',
configuration_secure_version='$GO_REVISION_{}_SECURE'.format(
config['edx_deployment'].upper()),
configuration_internal_version='$GO_REVISION_{}_SECURE'.format(
config['edx_deployment'].upper()),
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
edxapp_theme_source_repo=config['theme_url'],
edxapp_theme_version='$GO_REVISION_EDX_MICROSITE',
)
#
# Create the DB migration running stage.
#
ansible_inventory_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'ansible_inventory')
instance_ssh_key_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'key.pem')
launch_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'launch_info.yml')
for sub_app in config['edxapp_subapps']:
stages.generate_run_migrations(
pipeline,
db_migration_pass=config['db_migration_pass'],
inventory_location=ansible_inventory_location,
instance_key_location=instance_ssh_key_location,
launch_info_location=launch_info_location,
application_user=config['db_migration_user'],
application_name=config['play_name'],
application_path=config['application_path'],
sub_application_name=sub_app)
#
# Create the stage to deploy the AMI.
#
ami_file_location = utils.ArtifactLocation(pipeline.name,
constants.BUILD_AMI_STAGE_NAME,
constants.BUILD_AMI_JOB_NAME,
'ami.yml')
stages.generate_deploy_ami(
pipeline,
config['asgard_api_endpoints'],
config['asgard_token'],
config['aws_access_key_id'],
config['aws_secret_access_key'],
ami_file_location,
manual_approval=not config.get('auto_deploy_ami', False))
#
# Create the stage to terminate the EC2 instance used to both build the AMI and run DB migrations.
#
instance_info_location = utils.ArtifactLocation(
pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME,
constants.LAUNCH_INSTANCE_JOB_NAME, 'launch_info.yml')
stages.generate_terminate_instance(
pipeline,
instance_info_location,
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
hipchat_auth_token=config['hipchat_token'],
runif='any')
gcc.save_updated_config(save_config_locally=save_config_locally,
dry_run=dry_run)