def generate_rollback_asgs(stage, edp, deployment_artifact_location, config): """ Generates a job for rolling back ASGs (code). Args: stage (gomatic.gocd.pipelines.Stage): Stage to which this job belongs. edp (EDP): The EDP that this job should roll back. deployment_artifact_location (edxpipelines.utils.ArtifactLocation): Where to find the AMI artifact to roll back. config (dict): Environment-independent secure config. Returns: gomatic.gocd.pipelines.Job """ job = stage.ensure_job(constants.ROLLBACK_ASGS_JOB_NAME_TPL(edp)) tasks.generate_package_install(job, 'tubular') tasks.generate_target_directory(job) # Retrieve build info from the upstream deploy stage. tasks.retrieve_artifact(deployment_artifact_location, job) deployment_artifact_path = path_to_artifact(deployment_artifact_location.file_name) tasks.generate_rollback_asg( job, deployment_artifact_path, config['asgard_api_endpoints'], config['asgard_token'], config['aws_access_key_id'], config['aws_secret_access_key'], ) return job
def generate_fetch_tag_name(stage, acquia_env): """ Creates a job that fetches the tag name from a Drupal environment. Arguments: stage (gomatic.Stage): The stage to which the job is being added. acquia_env (str): The name of the environment from which the tag name should be fetched. Returns: gomatic.gocd.pipelines.Job: The new job. """ fetch_tag_job = stage.ensure_job(constants.FETCH_TAG_JOB_NAME) fetch_tag_job.ensure_resource("edxMarketing") tasks.generate_package_install(fetch_tag_job, 'tubular') tasks.generate_target_directory(fetch_tag_job) path_name = '../target/{env}_tag_name.txt' marketing_tasks.generate_fetch_tag(fetch_tag_job, acquia_env, path_name) artifact_path = 'target/{tag_name}.txt'.format( tag_name=constants.ACQUIA_ENV_TAG_NAME.format(acquia_env=acquia_env) ) fetch_tag_job.ensure_artifacts(set([BuildArtifact(artifact_path)])) return fetch_tag_job
def generate_find_and_advance_release( pipeline, advance_pipeline_name, advance_pipeline_stage_name, gocd_user, gocd_password, gocd_url, slack_token, slack_room=constants.SLACK_ROOM ): """ Generates a stage used to find the next release to advance and "manually" advance it. Args: pipeline (gomatic.Pipeline): advance_pipeline_name (str): Name of pipeline to advance. advance_pipeline_stage_name (str): Name of stage within pipeline to advance. gocd_user (str): GoCD username gocd_password (str): GoCD user's password gocd_url (str): URL of the GoCD instance slack_token (str): Token used to authenticate to Slack. slack_room (str): Slack room to which to post notifications. Returns: gomatic.Stage """ stage = pipeline.ensure_stage(constants.RELEASE_ADVANCER_STAGE_NAME) job = stage.ensure_job(constants.RELEASE_ADVANCER_JOB_NAME) tasks.generate_package_install(job, 'tubular') # Add task to generate the directory where the artifact file will be written. tasks.generate_target_directory(job) pipeline.ensure_unencrypted_secure_environment_variables( { 'GOCD_PASSWORD': gocd_password } ) job.ensure_encrypted_environment_variables( { 'SLACK_TOKEN': slack_token, } ) tasks.generate_find_and_advance_release( job, gocd_user, gocd_url, advance_pipeline_name, advance_pipeline_stage_name, slack_room, out_file=constants.FIND_ADVANCE_PIPELINE_OUT_FILENAME ) return stage
def generate_build_value_stream_map_url(pipeline): """ Genrates a stage that is used to build and serialize the value_stream_map url for a pipeline. Generates 1 artifact: value_stream_map.yaml - YAML file that contains the value_stream_map URL. Args: pipeline (gomatic.Pipeline): Returns: gomatic.Stage """ # Create the stage stage = pipeline.ensure_stage(constants.BUILD_VALUE_STREAM_MAP_URL_STAGE_NAME) # Create the job job = stage.ensure_job(constants.BUILD_VALUE_STREAM_MAP_URL_JOB_NAME) # Add task to generate the directory where the value_stream_map.yaml file will be written. tasks.generate_target_directory(job) # Add task to upload the value_stream_map.yaml file. value_stream_map_filepath = '{artifact_path}/{file_name}'.format( artifact_path=constants.ARTIFACT_PATH, file_name=constants.VALUE_STREAM_MAP_FILENAME ) job.ensure_artifacts(set([value_stream_map_filepath])) # Script to generate the value_stream_map_url and write it out to value_stream_map.yaml script = ''' if [ -z "$GO_PIPELINE_NAME" ] || [ -z "$GO_PIPELINE_LABEL" ]; then echo "Error: missing GO_PIPELINE_NAME and/or GO_PIPELINE_LABEL" 1>&2 exit 1 fi printf -- '---\n- deploy_value_stream_map: "{base_url}/$GO_PIPELINE_NAME/$GO_PIPELINE_LABEL"' > {filepath} '''.format(base_url=constants.BASE_VALUE_STREAM_MAP_URL, filepath=value_stream_map_filepath) job.add_task( ExecTask( [ '/bin/bash', '-c', script ] ) ) return stage
def generate_deploy_to_acquia(pipeline, acquia_env, use_tag=False): """ Creates a stage that deploys to an Acquia environment. Arguments: pipeline (gomatic.Pipeline): The pipeline to which the stage is being added. acquia_env (str): The environment name to which to deploy. use_tag (bool): True if deploying off a tag, False otherwise. Returns: gomatic.Stage: The new stage. """ deploy_stage_for_acquia_env = pipeline.ensure_stage( constants.DEPLOY_ACQUIA_ENV_STAGE_NAME.format(acquia_env=acquia_env) ) deploy_job_for_acquia_env = deploy_stage_for_acquia_env.ensure_job( constants.DEPLOY_ACQUIA_ENV_JOB_NAME.format(acquia_env=acquia_env) ) deploy_job_for_acquia_env.ensure_resource("edxMarketing") if use_tag: tasks.generate_target_directory(deploy_job_for_acquia_env) # fetch the tag name constants.new_tag_name_artifact_params = { 'pipeline': pipeline.name, 'stage': constants.BUILD_AND_PUSH_TO_ACQUIA_STAGE_NAME, 'job': constants.BUILD_AND_PUSH_TO_ACQUIA_JOB_NAME, 'src': FetchArtifactFile('deploy_tag_name.txt'), 'dest': 'target' } deploy_job_for_acquia_env.add_task(FetchArtifactTask(**constants.new_tag_name_artifact_params)) deploy_source = '$(cat ../{artifact_path}/deploy_tag_name.txt)'.format( artifact_path=constants.ARTIFACT_PATH ) else: deploy_source = '$MARKETING_REPOSITORY_VERSION' tasks.generate_package_install(deploy_job_for_acquia_env, 'tubular') marketing_tasks.generate_drupal_deploy( deploy_job_for_acquia_env, acquia_env, deploy_source ) return deploy_stage_for_acquia_env
def generate_rollback_in_acquia(pipeline, acquia_env, source_pipeline_env): """ Creates a stage that rolls back an Acquia environment. NOTE: Rollback is only implemented for tag based deployments. Arguments: pipeline (gomatic.Pipeline): The pipeline to which the stage is being added. acquia_env (str): The environment name to which to deploy. source_pipeline_env (str): The pipeline environment from which the rollback is sourced (e.g. 'prod'). Returns: gomatic.Stage: The new stage. """ rollback_stage = pipeline.ensure_stage(constants.ROLLBACK_STAGE_NAME.format(acquia_env=acquia_env)) rollback_stage.set_has_manual_approval() rollback_job = rollback_stage.ensure_job(constants.ROLLBACK_JOB_NAME.format(acquia_env=acquia_env)) rollback_job.ensure_resource("edxMarketing") rollback_tag_name = constants.ACQUIA_ENV_TAG_NAME.format(acquia_env=acquia_env) rollback_tag_name_artifact_params = { 'pipeline': constants.DEPLOY_MARKETING_PIPELINE_NAME.format(acquia_env=source_pipeline_env), 'stage': constants.FETCH_TAG_STAGE_NAME, 'job': constants.FETCH_TAG_JOB_NAME, 'src': FetchArtifactFile('{rollback_tag_name}.txt'.format(rollback_tag_name=rollback_tag_name)), 'dest': 'target' } tasks.generate_package_install(rollback_job, 'tubular') tasks.generate_target_directory(rollback_job) rollback_job.add_task(FetchArtifactTask(**rollback_tag_name_artifact_params)) marketing_tasks.generate_drupal_deploy( rollback_job, acquia_env, '$(cat ../{artifact_path}/{rollback_tag_name}.txt)'.format( artifact_path=constants.ARTIFACT_PATH, rollback_tag_name=rollback_tag_name ) ) return rollback_stage
def generate_build_and_push_to_acquia(pipeline, create_tag=False): """ Creates a stage that builds static assets and then pushes source and assets to Acquia. Arguments: pipeline (gomatic.Pipeline): The pipeline to which the stage is being added. create_tag (bool): True if a tag should first be created, False to push a branch directly. Returns: gomatic.Stage: The new stage. """ build_and_push_to_acquia_stage = pipeline.ensure_stage( constants.BUILD_AND_PUSH_TO_ACQUIA_STAGE_NAME ) build_and_push_to_acquia_job = build_and_push_to_acquia_stage.ensure_job( constants.BUILD_AND_PUSH_TO_ACQUIA_JOB_NAME ) build_and_push_to_acquia_job.ensure_resource("edxMarketing") marketing_tasks.setup_git_config(build_and_push_to_acquia_job) marketing_tasks.add_acquia_remote_if_needed(build_and_push_to_acquia_job) if create_tag: # Ensures the tag name is accessible in future jobs. build_and_push_to_acquia_job.ensure_artifacts(set([BuildArtifact('target/deploy_tag_name.txt')])) tasks.generate_package_install(build_and_push_to_acquia_job, 'tubular') tasks.generate_target_directory(build_and_push_to_acquia_job) marketing_tasks.create_tag_without_static_assets(build_and_push_to_acquia_job) marketing_tasks.build_and_commit_static_assets(build_and_push_to_acquia_job) marketing_tasks.create_and_push_tag_with_static_assets_to_acquia(build_and_push_to_acquia_job) marketing_tasks.push_original_tag_to_mktg(build_and_push_to_acquia_job) else: build_and_push_to_acquia_stage.set_has_manual_approval() marketing_tasks.build_and_commit_static_assets(build_and_push_to_acquia_job) marketing_tasks.push_branch_to_acquia(build_and_push_to_acquia_job) return build_and_push_to_acquia_stage
def generate_merge_release_candidate( pipeline, stage, token, org, repo, target_branch, head_sha, fast_forward_only, reference_repo=None, ): """ Generates a job that is used to merge a Git source branch into a target branch, optionally ensuring that the merge is a fast-forward merge. Args: pipeline (gomatic.Pipeline): The pipeline containing ``stage``. stage (gomatic.Stage): The stage to add the job to org (str): Name of the github organization that holds the repository (e.g. edx) repo (str): Name of repository (e.g edx-platform) target_branch (str): Name of the branch into which to merge the source branch head_sha (str): commit SHA or environment variable holding the SHA to tag as the release token (str): the github token used to create all these things. Will be an env_var 'GIT_TOKEN' fast_forward_only (bool): If True, force a fast-forward merge or fail. Returns: gomatic.Job """ merge_branch_job = stage.ensure_job(constants.GIT_MERGE_RC_BRANCH_JOB_NAME) tasks.generate_package_install(merge_branch_job, 'tubular') tasks.generate_target_directory(merge_branch_job) tasks.generate_merge_branch( pipeline, merge_branch_job, token, org, repo, head_sha, target_branch, fast_forward_only, reference_repo=reference_repo, ) return merge_branch_job
def generate_build_ami(stage, edp, app_repo_url, configuration_secure_material, configuration_internal_material, playbook_path, config, version_tags=None, **kwargs): """ Generates a job for creating a new AMI. Args: stage (gomatic.gocd.pipelines.Stage): Stage to which this job belongs. edp (edxpipelines.utils.EDP): Tuple indicating environment, deployment, and play for which an AMI will be created. app_repo_url (str): App repo's URL. configuration_secure_material (gomatic.gomatic.gocd.materials.GitMaterial): Secure configuration material. Destination directory expected to be 'configuration-secure'. configuration_internal_material (gomatic.gomatic.gocd.materials.GitMaterial): Internal configuration material. Destination directory expected to be 'configuration-internal'. playbook_path (str): Path to the Ansible playbook to run when creating the AMI. config (dict): Environment-specific secure config. version_tags (dict): An optional {app_name: (repo, version), ...} dict that specifies what versions to tag the AMI with. Returns: gomatic.gocd.pipelines.Job """ job = stage.ensure_job(constants.BUILD_AMI_JOB_NAME_TPL(edp)) tasks.generate_requirements_install(job, 'configuration') tasks.generate_package_install(job, 'tubular') tasks.generate_target_directory(job) # Locate the base AMI. tasks.generate_base_ami_selection( job, config['aws_access_key_id'], config['aws_secret_access_key'], edp=edp ) # Launch a new instance on which to build the AMI. tasks.generate_launch_instance( job, aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], ec2_vpc_subnet_id=config['ec2_vpc_subnet_id'], ec2_security_group_id=config['ec2_security_group_id'], ec2_instance_profile_name=config['ec2_instance_profile_name'], variable_override_path=path_to_artifact(constants.BASE_AMI_OVERRIDE_FILENAME), ) # Run the Ansible play for the service. tasks.generate_run_app_playbook( job, playbook_path, edp, app_repo_url, private_github_key=config['github_private_key'], hipchat_token=config['hipchat_token'], configuration_secure_dir=configuration_secure_material.destination_directory, configuration_internal_dir=configuration_internal_material.destination_directory, disable_edx_services='true', COMMON_TAG_EC2_INSTANCE='true', **kwargs ) # Create an AMI from the instance. tasks.generate_create_ami( job, edp.play, edp.deployment, edp.environment, app_repo_url, config['aws_access_key_id'], config['aws_secret_access_key'], path_to_artifact(constants.LAUNCH_INSTANCE_FILENAME), hipchat_token=config['hipchat_token'], version_tags=version_tags, **kwargs ) tasks.generate_ami_cleanup(job, config['hipchat_token'], runif='any') return job
def generate_poll_tests_and_merge_pr(pipeline, stage, job, stage_name, job_name, pr_artifact_params, artifact_filename, org, repo, token, initial_poll_wait, max_poll_tries, poll_interval, manual_approval): """ Generates a stage that is used to: - poll for successful completion of PR tests - merge the PR Args: pipeline (gomatic.Pipeline): Pipeline to attach this stage to stage (gomatic.Stage): Stage to use when adding tasks -or- None job (gomatic.Job): Job to use when adding tasks -or- None stage_name (str): Name of the stage job_name (str): Name of the job pr_artifact_params (dict): Params to use in creation of artifact-fetching task. artifact_filename (str): Filename of the artifact to fetch/read-in. org (str): Name of the github organization that holds the repository (e.g. edx) repo (str): Name of repository (e.g edx-platform) token (str): the github token used to create all these things. Will be an env_var 'GIT_TOKEN' initial_poll_wait (int): Number of seconds that will pass between 1st/2nd poll attempts. max_poll_tries (int): Maximum number of poll attempts that should occur before failing. poll_interval (int): Number of seconds between all poll attempts (after the 1st/2nd attempt interval). manual_approval (bool): Should this stage require manual approval? Returns: gomatic.Stage """ pipeline.ensure_environment_variables( { 'PR_TEST_INITIAL_WAIT_INTERVAL': str(initial_poll_wait), 'MAX_PR_TEST_POLL_TRIES': str(max_poll_tries), 'PR_TEST_POLL_INTERVAL': str(poll_interval) } ) pipeline.ensure_unencrypted_secure_environment_variables( { 'GIT_TOKEN': token } ) if stage is None: git_stage = pipeline.ensure_stage(stage_name) if manual_approval: git_stage.set_has_manual_approval() git_job = git_stage.ensure_job(job_name) else: git_stage = stage git_job = job tasks.generate_package_install(git_job, 'tubular') tasks.generate_target_directory(git_job) # Fetch the PR-creation material. git_job.add_task(FetchArtifactTask(**pr_artifact_params)) # Generate a task that poll the status of combined tests for a PR. tasks.generate_poll_pr_tests( git_job, org, repo, artifact_filename ) # Generate a task that merges a PR that has passed all its tests in the previous task. tasks.generate_merge_pr( git_job, org, repo, artifact_filename ) return git_stage
def generate_create_branch_and_pr(pipeline, stage_name, org, repo, source_branch, new_branch, target_branch, pr_title, pr_body, token, manual_approval): """ Generates a stage that is used to: - create a new branch off the HEAD of a source branch - create a PR to merge the new branch into a target branch Args: pipeline (gomatic.Pipeline): Pipeline to attach this stage to stage_name (str): Name of the stage org (str): Name of the github organization that holds the repository (e.g. edx) repo (str): Name of repository (e.g edx-platform) source_branch (str): Name of the branch to use in creating the new branch new_branch (str): Name of the branch to create off the HEAD of the source branch target_branch (str): Name of the branch into which to merge the source branch pr_title (str): Title of the new PR pr_body (str): Body of the new PR token (str): the github token used to create all these things. Will be an env_var 'GIT_TOKEN' manual_approval (bool): Should this stage require manual approval? Returns: gomatic.Stage """ git_stage = pipeline.ensure_stage(stage_name) if manual_approval: git_stage.set_has_manual_approval() git_job = git_stage.ensure_job(constants.CREATE_MASTER_MERGE_PR_JOB_NAME) tasks.generate_package_install(git_job, 'tubular') tasks.generate_target_directory(git_job) # Generate a task that creates a new branch off the HEAD of a source branch. tasks.generate_create_branch( pipeline, git_job, token, org, repo, target_branch=new_branch, source_branch=source_branch ) # Generate a task that creates a pull request merging the new branch from above into a target branch. tasks.generate_create_pr( git_job, org, repo, new_branch, target_branch, pr_title, pr_body ) return git_stage
def generate_run_migrations(pipeline, db_migration_pass, inventory_location, instance_key_location, launch_info_location, application_user, application_name, application_path, duration_threshold=None, from_address=None, to_addresses=None, sub_application_name=None, manual_approval=False): """ Generate the stage that applies/runs migrations. Args: pipeline (gomatic.Pipeline): Pipeline to which to add the run migrations stage. db_migration_pass (str): Password for the DB user used to run migrations. inventory_location (ArtifactLocation): Location of inventory containing the IP address of the EC2 instance, for fetching. instance_key_location (ArtifactLocation): Location of SSH key used to access the EC2 instance, for fetching. launch_info_location (ArtifactLocation): Location of the launch_info.yml file for fetching application_user (str): Username to use while running the migrations application_name (str): Name of the application (e.g. edxapp, ecommerce, etc...) application_path (str): path of the application installed on the target machine duration_threshold (int): Threshold in seconds over which a migration duration will be alerted. from_address (str): Any migration duration email alert will be from this address. to_addresses (list(str)): List of To: addresses for migration duration email alerts. sub_application_name (str): any sub application to insert in to the migrations commands {cms|lms} manual_approval (bool): Should this stage require manual approval? Returns: gomatic.Stage """ pipeline.ensure_environment_variables( { 'ARTIFACT_PATH': constants.ARTIFACT_PATH, 'ANSIBLE_CONFIG': constants.ANSIBLE_CONTINUOUS_DELIVERY_CONFIG } ) if duration_threshold: pipeline.ensure_environment_variables( { 'MAX_EMAIL_TRIES': constants.MAX_EMAIL_TRIES } ) if sub_application_name is not None: stage_name = "{}_{}".format(constants.APPLY_MIGRATIONS_STAGE, sub_application_name) else: stage_name = constants.APPLY_MIGRATIONS_STAGE stage = pipeline.ensure_stage(stage_name) if manual_approval: stage.set_has_manual_approval() job = stage.ensure_job(constants.APPLY_MIGRATIONS_JOB) tasks.generate_package_install(job, 'tubular') # Fetch the Ansible inventory to use in reaching the EC2 instance. tasks.retrieve_artifact(inventory_location, job, constants.ARTIFACT_PATH) # Fetch the SSH key to use in reaching the EC2 instance. tasks.retrieve_artifact(instance_key_location, job, constants.ARTIFACT_PATH) # ensure the target directoy exists tasks.generate_target_directory(job) # fetch the launch_info.yml tasks.retrieve_artifact(launch_info_location, job, constants.ARTIFACT_PATH) # The SSH key used to access the EC2 instance needs specific permissions. job.add_task( ExecTask( ['/bin/bash', '-c', 'chmod 600 {}'.format(instance_key_location.file_name)], working_dir=constants.ARTIFACT_PATH ) ) tasks.generate_requirements_install(job, 'configuration') tasks.generate_run_migrations( job, application_user, application_name, application_path, constants.DB_MIGRATION_USER, db_migration_pass, sub_application_name ) if duration_threshold: tasks.generate_check_migration_duration( job, application_name, constants.MIGRATION_RESULT_FILENAME, duration_threshold, from_address, to_addresses ) return stage
def install_pipelines(configurator, config): """ Install pipelines that can deploy the edx-mktg site. """ pipeline = configurator \ .ensure_pipeline_group(constants.DRUPAL_PIPELINE_GROUP_NAME) \ .ensure_replacement_of_pipeline(constants.DEPLOY_MARKETING_PIPELINE_NAME) \ .ensure_material(TUBULAR()) \ .ensure_material(EDX_MKTG()) \ .ensure_material(ECOM_SECURE()) pipeline.ensure_environment_variables( { 'MARKETING_REPOSITORY_VERSION': config['mktg_repository_version'], } ) pipeline.ensure_encrypted_environment_variables( { 'PRIVATE_GITHUB_KEY': config['github_private_key'], 'PRIVATE_ACQUIA_REMOTE': config['acquia_remote_url'], 'PRIVATE_ACQUIA_USERNAME': config['acquia_username'], 'PRIVATE_ACQUIA_PASSWORD': config['acquia_password'], 'PRIVATE_ACQUIA_GITHUB_KEY': config['acquia_github_key'] } ) # Stage to fetch the current tag names from stage and prod fetch_tag_stage = pipeline.ensure_stage(constants.FETCH_TAG_STAGE_NAME) fetch_tag_stage.set_has_manual_approval() fetch_tag_job = fetch_tag_stage.ensure_job(constants.FETCH_TAG_JOB_NAME) tasks.generate_package_install(fetch_tag_job, 'tubular') tasks.generate_target_directory(fetch_tag_job) path_name = '../target/{env}_tag_name.txt' tasks.generate_fetch_tag(fetch_tag_job, constants.STAGE_ENV, path_name) tasks.generate_fetch_tag(fetch_tag_job, constants.PROD_ENV, path_name) fetch_tag_job.ensure_artifacts( set([BuildArtifact('target/{stage_tag}.txt'.format(stage_tag=constants.STAGE_TAG_NAME)), BuildArtifact('target/{prod_tag}.txt'.format(prod_tag=constants.PROD_TAG_NAME))]) ) # Stage to create and push a tag to Acquia. push_to_acquia_stage = pipeline.ensure_stage(constants.PUSH_TO_ACQUIA_STAGE_NAME) push_to_acquia_job = push_to_acquia_stage.ensure_job(constants.PUSH_TO_ACQUIA_JOB_NAME) # Ensures the tag name is accessible in future jobs. push_to_acquia_job.ensure_artifacts( set([BuildArtifact('target/{new_tag}.txt'.format(new_tag=constants.NEW_TAG_NAME))]) ) tasks.generate_package_install(push_to_acquia_job, 'tubular') tasks.generate_target_directory(push_to_acquia_job) # Create a tag from MARKETING_REPOSITORY_VERSION branch of marketing repo push_to_acquia_job.add_task( tasks.bash_task( # Writing dates to a file should help with any issues dealing with a job # taking place over two days (23:59:59 -> 00:00:00). Only the day can be # affected since we don't use minutes or seconds. # NOTE: Uses UTC """\ echo -n "release-$(date +%Y-%m-%d-%H.%M)" > ../target/{new_tag}.txt && TAG_NAME=$(cat ../target/{new_tag}.txt) && /usr/bin/git config user.email "*****@*****.**" && /usr/bin/git config user.name "edx-secure" && /usr/bin/git tag -a $TAG_NAME -m "Release for $(date +%B\\ %d,\\ %Y). Created by $GO_TRIGGER_USER." && /usr/bin/git push origin $TAG_NAME """, new_tag=constants.NEW_TAG_NAME, working_dir='edx-mktg' ) ) # Set up Acquia remote repo and push tag to Acquia. Change new tag file to contain "tags/" for deployment. push_to_acquia_job.add_task( tasks.bash_task( """\ chmod 600 ../ecom-secure/acquia/acquia_github_key.pem && if [[ $(git remote) != *"acquia"* ]]; then /usr/bin/git remote add acquia $PRIVATE_ACQUIA_REMOTE ; fi && GIT_SSH_COMMAND="/usr/bin/ssh -o StrictHostKeyChecking=no -i ../{ecom_secure}/acquia/acquia_github_key.pem" /usr/bin/git push acquia $(cat ../target/{new_tag}.txt) && echo -n "tags/" | cat - ../target/{new_tag}.txt > temp && mv temp ../target/{new_tag}.txt """, new_tag=constants.NEW_TAG_NAME, ecom_secure=ECOM_SECURE().destination_directory, working_dir='edx-mktg' ) ) # Stage to backup database in stage backup_stage_database_stage = pipeline.ensure_stage(constants.BACKUP_STAGE_DATABASE_STAGE_NAME) backup_stage_database_job = backup_stage_database_stage.ensure_job(constants.BACKUP_STAGE_DATABASE_JOB_NAME) tasks.generate_package_install(backup_stage_database_job, 'tubular') tasks.generate_backup_drupal_database(backup_stage_database_job, constants.STAGE_ENV) # Stage to deploy to stage deploy_stage_for_stage = pipeline.ensure_stage(constants.DEPLOY_STAGE_STAGE_NAME) deploy_job_for_stage = deploy_stage_for_stage.ensure_job(constants.DEPLOY_STAGE_JOB_NAME) tasks.generate_package_install(deploy_job_for_stage, 'tubular') tasks.generate_target_directory(deploy_job_for_stage) # fetch the tag name constants.new_tag_name_artifact_params = { 'pipeline': constants.DEPLOY_MARKETING_PIPELINE_NAME, 'stage': constants.PUSH_TO_ACQUIA_STAGE_NAME, 'job': constants.PUSH_TO_ACQUIA_JOB_NAME, 'src': FetchArtifactFile('{new_tag}.txt'.format(new_tag=constants.NEW_TAG_NAME)), 'dest': 'target' } deploy_job_for_stage.add_task(FetchArtifactTask(**constants.new_tag_name_artifact_params)) tasks.generate_drupal_deploy( deploy_job_for_stage, constants.STAGE_ENV, '{new_tag}.txt'.format(new_tag=constants.NEW_TAG_NAME) ) # Stage to clear caches in stage clear_stage_caches_stage = pipeline.ensure_stage(constants.CLEAR_STAGE_CACHES_STAGE_NAME) clear_stage_caches_job = clear_stage_caches_stage.ensure_job(constants.CLEAR_STAGE_CACHES_JOB_NAME) tasks.generate_package_install(clear_stage_caches_job, 'tubular') clear_stage_caches_job.add_task( tasks.bash_task( """ chmod 600 ecom-secure/acquia/acquia_github_key.pem && cp {ecom_secure}/acquia/acquia_github_key.pem {edx_mktg}/docroot/ """, ecom_secure=ECOM_SECURE().destination_directory, edx_mktg=EDX_MKTG().destination_directory ) ) tasks.generate_flush_drupal_caches(clear_stage_caches_job, constants.STAGE_ENV) tasks.generate_clear_varnish_cache(clear_stage_caches_job, constants.STAGE_ENV) # Stage to backup database in prod backup_prod_database_stage = pipeline.ensure_stage(constants.BACKUP_PROD_DATABASE_STAGE_NAME) backup_prod_database_stage.set_has_manual_approval() backup_prod_database_job = backup_prod_database_stage.ensure_job(constants.BACKUP_PROD_DATABASE_JOB_NAME) tasks.generate_package_install(backup_prod_database_job, 'tubular') tasks.generate_backup_drupal_database(backup_prod_database_job, constants.PROD_ENV) # Stage to deploy to prod deploy_stage_for_prod = pipeline.ensure_stage(constants.DEPLOY_PROD_STAGE_NAME) deploy_job_for_prod = deploy_stage_for_prod.ensure_job(constants.DEPLOY_PROD_JOB_NAME) tasks.generate_package_install(deploy_job_for_prod, 'tubular') tasks.generate_target_directory(deploy_job_for_prod) deploy_job_for_prod.add_task(FetchArtifactTask(**constants.new_tag_name_artifact_params)) tasks.generate_drupal_deploy( deploy_job_for_prod, constants.PROD_ENV, '{new_tag}.txt'.format(new_tag=constants.NEW_TAG_NAME) ) # Stage to clear caches in prod clear_prod_caches_stage = pipeline.ensure_stage(constants.CLEAR_PROD_CACHES_STAGE_NAME) clear_prod_caches_job = clear_prod_caches_stage.ensure_job(constants.CLEAR_PROD_CACHES_JOB_NAME) tasks.generate_package_install(clear_prod_caches_job, 'tubular') clear_prod_caches_job.add_task( tasks.bash_task( """ chmod 600 ecom-secure/acquia/acquia_github_key.pem && cp {ecom_secure}/acquia/acquia_github_key.pem {edx_mktg}/docroot/ """, ecom_secure=ECOM_SECURE().destination_directory, edx_mktg=EDX_MKTG().destination_directory ) ) tasks.generate_flush_drupal_caches(clear_prod_caches_job, constants.PROD_ENV) tasks.generate_clear_varnish_cache(clear_prod_caches_job, constants.PROD_ENV)
def generate_ansible_stage(stage_name, task, pipeline, inventory_location, instance_key_location, launch_info_location, application_user, application_name, application_path, hipchat_auth_token, hipchat_room=constants.HIPCHAT_ROOM, manual_approval=False): """ Generate the stage with the given name, that runs the specified task. Args: stage_name (str): Name of the generated stage. task (function): Task to be executed by the stage. pipeline (gomatic.Pipeline): Pipeline to which to add the run migrations stage. inventory_location (ArtifactLocation): Location of inventory containing the IP address of the EC2 instance, for fetching. instance_key_location (ArtifactLocation): Location of SSH key used to access the EC2 instance, for fetching. launch_info_location (ArtifactLocation): Location of the launch_info.yml file for fetching application_user (str): Username to use while running the migrations application_name (str): Name of the application (e.g. edxapp, programs, etc...) application_path (str): path of the application installed on the target machine hipchat_auth_token (str): HipChat authentication token hipchat_room (str): HipChat room where announcements should be made manual_approval (bool): Should this stage require manual approval? Returns: gomatic.Stage """ pipeline.ensure_environment_variables({ 'APPLICATION_USER': application_user, 'APPLICATION_NAME': application_name, 'APPLICATION_PATH': application_path, 'HIPCHAT_ROOM': hipchat_room, }) pipeline.ensure_encrypted_environment_variables({ 'HIPCHAT_TOKEN': hipchat_auth_token, }) stage = pipeline.ensure_stage(stage_name) if manual_approval: stage.set_has_manual_approval() job = stage.ensure_job(stage_name + '_job') # Fetch the Ansible inventory to use in reaching the EC2 instance. artifact_params = { "pipeline": inventory_location.pipeline, "stage": inventory_location.stage, "job": inventory_location.job, "src": FetchArtifactFile(inventory_location.file_name), "dest": 'configuration' } job.add_task(FetchArtifactTask(**artifact_params)) # Fetch the SSH key to use in reaching the EC2 instance. artifact_params = { "pipeline": instance_key_location.pipeline, "stage": instance_key_location.stage, "job": instance_key_location.job, "src": FetchArtifactFile(instance_key_location.file_name), "dest": 'configuration' } job.add_task(FetchArtifactTask(**artifact_params)) # ensure the target directoy exists tasks.generate_target_directory(job) # fetch the launch_info.yml artifact_params = { "pipeline": launch_info_location.pipeline, "stage": launch_info_location.stage, "job": launch_info_location.job, "src": FetchArtifactFile(launch_info_location.file_name), "dest": "target" } job.add_task(FetchArtifactTask(**artifact_params)) # The SSH key used to access the EC2 instance needs specific permissions. job.add_task( ExecTask([ '/bin/bash', '-c', 'chmod 600 {}'.format( instance_key_location.file_name) ], working_dir='configuration')) tasks.generate_requirements_install(job, 'configuration') task(job) return stage
def generate_create_ami_from_instance( pipeline, play, deployment, edx_environment, app_repo, configuration_secure_repo, aws_access_key_id, aws_secret_access_key, configuration_repo=constants.PUBLIC_CONFIGURATION_REPO_URL, ami_creation_timeout="3600", ami_wait='yes', cache_id='', artifact_path=constants.ARTIFACT_PATH, hipchat_room=constants.HIPCHAT_ROOM, manual_approval=False, **kwargs): """ Generates an artifact ami.yml: ami_id: ami-abcdefg ami_message: AMI creation operation complete ami_state: available Args: pipeline (gomatic.Pipeline): play (str): Play that was run on the instance (used for tagging) deployment (str): edx_environment (str): app_repo (str): configuration_secure_repo (str): aws_access_key_id (str): aws_secret_access_key (str): configuration_repo (str): ami_creation_timeout (str): ami_wait (str): cache_id (str): artifact_path (str): hipchat_room (str): manual_approval (bool): **kwargs (dict): k,v pairs: k: the name of the option to pass to ansible v: the value to use for this option Returns: gomatic.Stage """ stage = pipeline.ensure_stage(constants.BUILD_AMI_STAGE_NAME) if manual_approval: stage.set_has_manual_approval() pipeline.ensure_encrypted_environment_variables({ 'AWS_ACCESS_KEY_ID': aws_access_key_id, 'AWS_SECRET_ACCESS_KEY': aws_secret_access_key }) pipeline.ensure_environment_variables({ 'PLAY': play, 'DEPLOYMENT': deployment, 'EDX_ENVIRONMENT': edx_environment, 'APP_REPO': app_repo, 'CONFIGURATION_REPO': configuration_repo, 'CONFIGURATION_SECURE_REPO': configuration_secure_repo, 'AMI_CREATION_TIMEOUT': ami_creation_timeout, 'AMI_WAIT': ami_wait, 'CACHE_ID': cache_id, # gocd build number 'ARTIFACT_PATH': artifact_path, 'HIPCHAT_ROOM': hipchat_room, 'ANSIBLE_CONFIG': constants.ANSIBLE_CONTINUOUS_DELIVERY_CONFIG, }) # Install the requirements. job = stage.ensure_job(constants.BUILD_AMI_JOB_NAME) tasks.generate_requirements_install(job, 'tubular') tasks.generate_requirements_install(job, 'configuration') tasks.generate_target_directory(job) # fetch the key material artifact_params = { 'pipeline': pipeline.name, 'stage': constants.LAUNCH_INSTANCE_STAGE_NAME, 'job': constants.LAUNCH_INSTANCE_JOB_NAME, 'src': FetchArtifactFile("launch_info.yml"), 'dest': constants.ARTIFACT_PATH } job.add_task(FetchArtifactTask(**artifact_params)) # Create an AMI from the instance tasks.generate_create_ami(job, **kwargs) return stage
def install_pipelines(configurator, config): """ Install pipelines that can rollback the stage edx-mktg site. """ pipeline = configurator \ .ensure_pipeline_group(constants.DRUPAL_PIPELINE_GROUP_NAME) \ .ensure_replacement_of_pipeline('rollback-stage-marketing-site') \ .ensure_material(TUBULAR()) \ .ensure_material(EDX_MKTG()) \ .ensure_material(ECOM_SECURE()) \ .ensure_material(PipelineMaterial(constants.DEPLOY_MARKETING_PIPELINE_NAME, constants.FETCH_TAG_STAGE_NAME)) pipeline.ensure_environment_variables( { 'MARKETING_REPOSITORY_VERSION': config['mktg_repository_version'], } ) pipeline.ensure_encrypted_environment_variables( { 'PRIVATE_GITHUB_KEY': config['github_private_key'], 'PRIVATE_ACQUIA_USERNAME': config['acquia_username'], 'PRIVATE_ACQUIA_PASSWORD': config['acquia_password'], 'PRIVATE_ACQUIA_GITHUB_KEY': config['acquia_github_key'], } ) stage_tag_name_artifact_params = { 'pipeline': constants.DEPLOY_MARKETING_PIPELINE_NAME, 'stage': constants.FETCH_TAG_STAGE_NAME, 'job': constants.FETCH_TAG_JOB_NAME, 'src': FetchArtifactFile('{stage_tag}.txt'.format(stage_tag=constants.STAGE_TAG_NAME)), 'dest': 'target' } # Stage to rollback stage to its last stable tag rollback_stage = pipeline.ensure_stage(constants.ROLLBACK_STAGE_NAME) rollback_stage.set_has_manual_approval() rollback_job = rollback_stage.ensure_job(constants.ROLLBACK_JOB_NAME) tasks.generate_package_install(rollback_job, 'tubular') tasks.generate_target_directory(rollback_job) rollback_job.add_task(FetchArtifactTask(**stage_tag_name_artifact_params)) tasks.generate_drupal_deploy( rollback_job, constants.STAGE_ENV, '{stage_tag}.txt'.format(stage_tag=constants.STAGE_TAG_NAME) ) # Stage to clear the caches clear_stage_caches_stage = pipeline.ensure_stage(constants.CLEAR_STAGE_CACHES_STAGE_NAME) clear_stage_caches_job = clear_stage_caches_stage.ensure_job(constants.CLEAR_STAGE_CACHES_JOB_NAME) tasks.generate_package_install(clear_stage_caches_job, 'tubular') clear_stage_caches_job.add_task( tasks.bash_task( """ chmod 600 ecom-secure/acquia/acquia_github_key.pem && cp {ecom_secure}/acquia/acquia_github_key.pem {edx_mktg}/docroot/ """, ecom_secure=ECOM_SECURE().destination_directory, edx_mktg=EDX_MKTG().destination_directory ) ) tasks.generate_flush_drupal_caches(clear_stage_caches_job, constants.STAGE_ENV) tasks.generate_clear_varnish_cache(clear_stage_caches_job, constants.STAGE_ENV)
def install_pipeline(save_config_locally, dry_run, variable_files, cmd_line_vars): config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, )) configurator = GoCdConfigurator( HostRestClient(config['gocd_url'], config['gocd_username'], config['gocd_password'], ssl=True)) pipeline = configurator \ .ensure_pipeline_group(DRUPAL_PIPELINE_GROUP_NAME) \ .ensure_replacement_of_pipeline(DEPLOY_MARKETING_PIPELINE_NAME) \ .set_git_material(GitMaterial('https://github.com/edx/tubular', polling=True, destination_directory='tubular', ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX ) ) pipeline.ensure_environment_variables( { 'MARKETING_REPOSITORY_VERSION': config['mktg_repository_version'], } ) pipeline.ensure_encrypted_environment_variables( { 'PRIVATE_GITHUB_KEY': config['github_private_key'], 'PRIVATE_MARKETING_REPOSITORY_URL': config['mktg_repository_url'], 'PRIVATE_ACQUIA_REMOTE': config['acquia_remote_url'], 'PRIVATE_ACQUIA_USERNAME': config['acquia_username'], 'PRIVATE_ACQUIA_PASSWORD': config['acquia_password'], 'PRIVATE_ACQUIA_GITHUB_KEY': config['acquia_github_key'] } ) # Stage to fetch the current tag names from stage and prod fetch_tag_stage = pipeline.ensure_stage(FETCH_TAG_STAGE_NAME) fetch_tag_stage.set_has_manual_approval() fetch_tag_job = fetch_tag_stage.ensure_job(FETCH_TAG_JOB_NAME) tasks.generate_requirements_install(fetch_tag_job, 'tubular') tasks.generate_target_directory(fetch_tag_job) path_name = '../target/{env}_tag_name.txt' tasks.generate_fetch_tag(fetch_tag_job, STAGE_ENV, path_name) tasks.generate_fetch_tag(fetch_tag_job, PROD_ENV, path_name) fetch_tag_job.ensure_artifacts( set([BuildArtifact('target/{stage_tag}.txt'.format(stage_tag=STAGE_TAG_NAME)), BuildArtifact('target/{prod_tag}.txt'.format(prod_tag=PROD_TAG_NAME))]) ) # Stage to create and push a tag to Acquia. push_to_acquia_stage = pipeline.ensure_stage(PUSH_TO_ACQUIA_STAGE_NAME) push_to_acquia_job = push_to_acquia_stage.ensure_job(PUSH_TO_ACQUIA_JOB_NAME) # Ensures the tag name is accessible in future jobs. push_to_acquia_job.ensure_artifacts( set([BuildArtifact('target/{new_tag}.txt'.format(new_tag=NEW_TAG_NAME))]) ) tasks.generate_requirements_install(push_to_acquia_job, 'tubular') tasks.generate_target_directory(push_to_acquia_job) tasks.fetch_edx_mktg(push_to_acquia_job, 'edx-mktg') # Create a tag from MARKETING_REPOSITORY_VERSION branch of marketing repo push_to_acquia_job.add_task( ExecTask( [ '/bin/bash', '-c', # Writing dates to a file should help with any issues dealing with a job # taking place over two days (23:59:59 -> 00:00:00). Only the day can be # affected since we don't use minutes or seconds. # NOTE: Uses UTC 'echo -n "release-$(date +%Y-%m-%d-%H.%M)" > ../target/{new_tag}.txt && ' 'TAG_NAME=$(cat ../target/{new_tag}.txt) && ' '/usr/bin/git config user.email "*****@*****.**" && ' '/usr/bin/git config user.name "edx-secure" && ' '/usr/bin/git tag -a $TAG_NAME -m "Release for $(date +%B\ %d,\ %Y). Created by $GO_TRIGGER_USER." && ' 'GIT_SSH_COMMAND="/usr/bin/ssh -o StrictHostKeyChecking=no -i ../github_key.pem" ' '/usr/bin/git push origin $TAG_NAME'.format(new_tag=NEW_TAG_NAME) ], working_dir='edx-mktg' ) ) # Set up Acquia Github key for use in pushing tag to Acquia tasks.format_RSA_key(push_to_acquia_job, 'acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY') # Set up Acquia remote repo and push tag to Acquia. Change new tag file to contain "tags/" for deployment. push_to_acquia_job.add_task( ExecTask( [ '/bin/bash', '-c', '/usr/bin/git remote add acquia $PRIVATE_ACQUIA_REMOTE && ' 'GIT_SSH_COMMAND="/usr/bin/ssh -o StrictHostKeyChecking=no -i ../acquia_github_key.pem" ' '/usr/bin/git push acquia $(cat ../target/{new_tag}.txt) && ' 'echo -n "tags/" | cat - ../target/{new_tag}.txt > temp && mv temp ../target/{new_tag}.txt'.format(new_tag=NEW_TAG_NAME) ], working_dir='edx-mktg' ) ) # Stage to backup database in stage backup_stage_database_stage = pipeline.ensure_stage(BACKUP_STAGE_DATABASE_STAGE_NAME) backup_stage_database_job = backup_stage_database_stage.ensure_job(BACKUP_STAGE_DATABASE_JOB_NAME) tasks.generate_requirements_install(backup_stage_database_job, 'tubular') tasks.generate_backup_drupal_database(backup_stage_database_job, STAGE_ENV) # Stage to deploy to stage deploy_stage_for_stage = pipeline.ensure_stage(DEPLOY_STAGE_STAGE_NAME) deploy_job_for_stage = deploy_stage_for_stage.ensure_job(DEPLOY_STAGE_JOB_NAME) tasks.generate_requirements_install(deploy_job_for_stage, 'tubular') tasks.generate_target_directory(deploy_job_for_stage) # fetch the tag name new_tag_name_artifact_params = { 'pipeline': DEPLOY_MARKETING_PIPELINE_NAME, 'stage': PUSH_TO_ACQUIA_STAGE_NAME, 'job': PUSH_TO_ACQUIA_JOB_NAME, 'src': FetchArtifactFile('{new_tag}.txt'.format(new_tag=NEW_TAG_NAME)), 'dest': 'target' } deploy_job_for_stage.add_task(FetchArtifactTask(**new_tag_name_artifact_params)) tasks.generate_drupal_deploy(deploy_job_for_stage, STAGE_ENV, '{new_tag}.txt'.format(new_tag=NEW_TAG_NAME)) # Stage to clear caches in stage clear_stage_caches_stage = pipeline.ensure_stage(CLEAR_STAGE_CACHES_STAGE_NAME) clear_stage_caches_job = clear_stage_caches_stage.ensure_job(CLEAR_STAGE_CACHES_JOB_NAME) tasks.fetch_edx_mktg(clear_stage_caches_job, 'edx-mktg') tasks.generate_requirements_install(clear_stage_caches_job, 'tubular') tasks.format_RSA_key(clear_stage_caches_job, 'edx-mktg/docroot/acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY') tasks.generate_flush_drupal_caches(clear_stage_caches_job, STAGE_ENV) tasks.generate_clear_varnish_cache(clear_stage_caches_job, STAGE_ENV) # Stage to backup database in prod backup_prod_database_stage = pipeline.ensure_stage(BACKUP_PROD_DATABASE_STAGE_NAME) backup_prod_database_stage.set_has_manual_approval() backup_prod_database_job = backup_prod_database_stage.ensure_job(BACKUP_PROD_DATABASE_JOB_NAME) tasks.generate_requirements_install(backup_prod_database_job, 'tubular') tasks.generate_backup_drupal_database(backup_prod_database_job, PROD_ENV) # Stage to deploy to prod deploy_stage_for_prod = pipeline.ensure_stage(DEPLOY_PROD_STAGE_NAME) deploy_job_for_prod = deploy_stage_for_prod.ensure_job(DEPLOY_PROD_JOB_NAME) tasks.generate_requirements_install(deploy_job_for_prod, 'tubular') tasks.generate_target_directory(deploy_job_for_prod) deploy_job_for_prod.add_task(FetchArtifactTask(**new_tag_name_artifact_params)) tasks.generate_drupal_deploy(deploy_job_for_prod, PROD_ENV, '{new_tag}.txt'.format(new_tag=NEW_TAG_NAME)) # Stage to clear caches in prod clear_prod_caches_stage = pipeline.ensure_stage(CLEAR_PROD_CACHES_STAGE_NAME) clear_prod_caches_job = clear_prod_caches_stage.ensure_job(CLEAR_PROD_CACHES_JOB_NAME) tasks.fetch_edx_mktg(clear_prod_caches_job, 'edx-mktg') tasks.generate_requirements_install(clear_prod_caches_job, 'tubular') tasks.format_RSA_key(clear_prod_caches_job, 'edx-mktg/docroot/acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY') tasks.generate_flush_drupal_caches(clear_prod_caches_job, PROD_ENV) tasks.generate_clear_varnish_cache(clear_prod_caches_job, PROD_ENV) configurator.save_updated_config(save_config_locally=save_config_locally, dry_run=dry_run)
def generate_rollback_migrations( stage, edp, application_user, application_name, application_path, db_migration_user, db_migration_pass, migration_info_location, inventory_location=None, instance_key_location=None, ami_artifact_location=None, config=None, sub_application_name=None, additional_migrations=None, ): """ Generates a job for rolling back database migrations. Args: stage (gomatic.gocd.pipelines.Stage): Stage this job will be part of edp (EDP): EDP that this job will roll back migration_info_location (edxpipelines.utils.ArtifactLocation): Location of the migration output to roll back inventory_location (edxpipelines.utils.ArtifactLocation): Location of the ansible inventory instance_key_location (edxpipelines.utils.ArtifactLocation): Location of the key used to ssh in to the instance ami_artifact_location (edxpipelines.utils.ArtifactLocation): AMI to use when launching instance used to roll back migrations. config (dict): Environment-specific secure config. sub_application_name (str): additional command to be passed to the migrate app {cms|lms} additional_migrations (list[edxpipelines.utils.MigrationAppInfo]): Additional applications to migrate. Will only run if has_migrations=True Returns: gomatic.gocd.pipelines.Job """ if not additional_migrations: additional_migrations = [] job_name = constants.ROLLBACK_MIGRATIONS_JOB_NAME_TPL(edp) if sub_application_name is not None: job_name += '_{}'.format(sub_application_name) job = stage.ensure_job(job_name) tasks.generate_requirements_install(job, 'configuration') tasks.generate_target_directory(job) is_instance_launch_required = ami_artifact_location and config if is_instance_launch_required: # Retrieve the AMI ID from the upstream build stage. tasks.retrieve_artifact(ami_artifact_location, job) variable_override_path = path_to_artifact(ami_artifact_location.file_name) tasks.generate_launch_instance( job, aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], ec2_vpc_subnet_id=config['ec2_vpc_subnet_id'], ec2_security_group_id=config['ec2_security_group_id'], ec2_instance_profile_name=config['ec2_instance_profile_name'], variable_override_path=variable_override_path, ) else: # The instance was launched elsewhere. Fetch the Ansible inventory to # use in reaching the EC2 instance. tasks.retrieve_artifact(inventory_location, job) # Fetch the SSH key to use in reaching the EC2 instance. tasks.retrieve_artifact(instance_key_location, job) # SSH key used to access the instance needs specific permissions. job.ensure_task(tasks.bash_task( 'chmod 600 {key_pem_path}', key_pem_path=path_to_artifact(constants.KEY_PEM_FILENAME) )) # Fetch the migration output. tasks.retrieve_artifact(migration_info_location, job) tasks.generate_migration_rollback( job=job, application_user=application_user, application_name=application_name, application_path=application_path, db_migration_user=db_migration_user, db_migration_pass=db_migration_pass, sub_application_name=sub_application_name, ) for migration_info in additional_migrations: tasks.generate_migration_rollback( job=job, application_user=migration_info.name, application_name=migration_info.name, application_path=migration_info.path, db_migration_user=db_migration_user, db_migration_pass=db_migration_pass, sub_application_name=migration_info.sub_application_name, ) # If an instance was launched as part of this job, clean it up. if is_instance_launch_required: tasks.generate_ami_cleanup(job, config['slack_token'], runif='any') return job
def generate_build_ami(stage, edp, app_repo_url, configuration_secure_material, configuration_internal_material, playbook_path, config, version_tags=None, **kwargs): """ Generates a job for creating a new AMI. Args: stage (gomatic.gocd.pipelines.Stage): Stage to which this job belongs. edp (edxpipelines.utils.EDP): Tuple indicating environment, deployment, and play for which an AMI will be created. app_repo_url (str): App repo's URL. configuration_secure_material (gomatic.gomatic.gocd.materials.GitMaterial): Secure configuration material. Destination directory expected to be 'configuration-secure'. configuration_internal_material (gomatic.gomatic.gocd.materials.GitMaterial): Internal configuration material. Destination directory expected to be 'configuration-internal'. playbook_path (str): Path to the Ansible playbook to run when creating the AMI. config (dict): Environment-specific secure config. version_tags (dict): An optional {app_name: (repo, version), ...} dict that specifies what versions to tag the AMI with. Returns: gomatic.gocd.pipelines.Job """ job = stage.ensure_job(constants.BUILD_AMI_JOB_NAME_TPL(edp)) tasks.generate_requirements_install(job, 'configuration') tasks.generate_package_install(job, 'tubular') tasks.generate_target_directory(job) # Locate the base AMI. tasks.generate_base_ami_selection( job, config['aws_access_key_id'], config['aws_secret_access_key'], edp=edp ) # Launch a new instance on which to build the AMI. tasks.generate_launch_instance( job, aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], ec2_vpc_subnet_id=config['ec2_vpc_subnet_id'], ec2_security_group_id=config['ec2_security_group_id'], ec2_instance_profile_name=config['ec2_instance_profile_name'], variable_override_path=path_to_artifact(constants.BASE_AMI_OVERRIDE_FILENAME), ) tasks.generate_ensure_python2(job) # Run the Ansible play for the service. tasks.generate_run_app_playbook( job, playbook_path, edp, app_repo_url, slack_token=config['slack_token'], configuration_secure_dir=configuration_secure_material.destination_directory, configuration_internal_dir=configuration_internal_material.destination_directory, disable_edx_services='true', COMMON_TAG_EC2_INSTANCE='true', **kwargs ) # Create an AMI from the instance. tasks.generate_create_ami( job, edp.play, edp.deployment, edp.environment, app_repo_url, config['aws_access_key_id'], config['aws_secret_access_key'], path_to_artifact(constants.LAUNCH_INSTANCE_FILENAME), slack_token=config['slack_token'], version_tags=version_tags, **kwargs ) tasks.generate_ami_cleanup(job, config['slack_token'], runif='any') return job
def generate_rollback_migrations( stage, edp, application_user, application_name, application_path, db_migration_user, db_migration_pass, migration_info_location, inventory_location=None, instance_key_location=None, ami_artifact_location=None, config=None, sub_application_name=None ): """ Generates a job for rolling back database migrations. Args: stage (gomatic.gocd.pipelines.Stage): Stage this job will be part of edp (EDP): EDP that this job will roll back migration_info_location (edxpipelines.utils.ArtifactLocation): Location of the migration output to roll back inventory_location (edxpipelines.utils.ArtifactLocation): Location of the ansible inventory instance_key_location (edxpipelines.utils.ArtifactLocation): Location of the key used to ssh in to the instance ami_artifact_location (edxpipelines.utils.ArtifactLocation): AMI to use when launching instance used to roll back migrations. config (dict): Environment-specific secure config. sub_application_name (str): additional command to be passed to the migrate app {cms|lms} Returns: gomatic.gocd.pipelines.Job """ job_name = constants.ROLLBACK_MIGRATIONS_JOB_NAME_TPL(edp) if sub_application_name is not None: job_name += '_{}'.format(sub_application_name) job = stage.ensure_job(job_name) tasks.generate_requirements_install(job, 'configuration') tasks.generate_target_directory(job) is_instance_launch_required = ami_artifact_location and config if is_instance_launch_required: # Retrieve the AMI ID from the upstream build stage. tasks.retrieve_artifact(ami_artifact_location, job) variable_override_path = path_to_artifact(ami_artifact_location.file_name) tasks.generate_launch_instance( job, aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], ec2_vpc_subnet_id=config['ec2_vpc_subnet_id'], ec2_security_group_id=config['ec2_security_group_id'], ec2_instance_profile_name=config['ec2_instance_profile_name'], variable_override_path=variable_override_path, ) else: # The instance was launched elsewhere. Fetch the Ansible inventory to # use in reaching the EC2 instance. tasks.retrieve_artifact(inventory_location, job) # Fetch the SSH key to use in reaching the EC2 instance. tasks.retrieve_artifact(instance_key_location, job) # SSH key used to access the instance needs specific permissions. job.ensure_task(tasks.bash_task( 'chmod 600 {key_pem_path}', key_pem_path=path_to_artifact(constants.KEY_PEM_FILENAME) )) # Fetch the migration output. tasks.retrieve_artifact(migration_info_location, job) tasks.generate_migration_rollback( job=job, application_user=application_user, application_name=application_name, application_path=application_path, db_migration_user=db_migration_user, db_migration_pass=db_migration_pass, sub_application_name=sub_application_name, ) # If an instance was launched as part of this job, clean it up. if is_instance_launch_required: tasks.generate_ami_cleanup(job, config['hipchat_token'], runif='any') return job
def generate_create_ami_from_instance(pipeline, edp, app_repo, aws_access_key_id, aws_secret_access_key, ami_creation_timeout=3600, ami_wait='yes', cache_id='', artifact_path=constants.ARTIFACT_PATH, slack_token='', slack_room=constants.SLACK_ROOM, manual_approval=False, version_tags=None, **kwargs): """ Generates an artifact ami.yml: ami_id: ami-abcdefg ami_message: AMI creation operation complete ami_state: available Args: pipeline (gomatic.Pipeline): edp (EDP): app_repo (str): configuration_secure_repo (str): aws_access_key_id (str): aws_secret_access_key (str): configuration_repo (str): ami_creation_timeout (str): ami_wait (str): cache_id (str): artifact_path (str): slack_token (str): Token used to authenticate to Slack. slack_room (str): Slack room to which to post notifications. manual_approval (bool): version_tags (dict): An optional {app_name: (repo, version), ...} dict that specifies what versions to tag the AMI with. **kwargs (dict): k,v pairs: k: the name of the option to pass to ansible v: the value to use for this option Returns: gomatic.Stage """ stage = pipeline.ensure_stage(constants.BUILD_AMI_STAGE_NAME) if manual_approval: stage.set_has_manual_approval() # Install the requirements. job = stage.ensure_job(constants.BUILD_AMI_JOB_NAME) tasks.generate_package_install(job, 'tubular') tasks.generate_requirements_install(job, 'configuration') tasks.generate_target_directory(job) launch_info_artifact = ArtifactLocation( pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME, constants.LAUNCH_INSTANCE_JOB_NAME, constants.LAUNCH_INSTANCE_FILENAME, ) tasks.retrieve_artifact(launch_info_artifact, job) # Create an AMI from the instance tasks.generate_create_ami( job=job, play=edp.play, deployment=edp.deployment, edx_environment=edp.environment, app_repo=app_repo, launch_info_path='{}/{}'.format(constants.ARTIFACT_PATH, constants.LAUNCH_INSTANCE_FILENAME), aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, ami_creation_timeout=ami_creation_timeout, ami_wait=ami_wait, cache_id=cache_id, artifact_path=artifact_path, slack_token=slack_token, slack_room=slack_room, version_tags=version_tags, **kwargs) return stage
def generate_run_play( pipeline, playbook_with_path, play, deployment, edx_environment, app_repo, private_github_key='', hipchat_token='', hipchat_room=constants.HIPCHAT_ROOM, manual_approval=False, configuration_secure_dir=constants.PRIVATE_CONFIGURATION_LOCAL_DIR, configuration_internal_dir=constants.INTERNAL_CONFIGURATION_LOCAL_DIR, **kwargs): """ TODO: This currently runs from the configuration/playbooks/continuous_delivery/ directory. Need to figure out how to pass in a configuration file to ansible-play correctly. TE-1608 Assumes: - generate_launch_instance stage was used launch the instance preceding this stage. - Requires the ansible_inventory and key.pem files to be in the constants.ARTIFACT_DIRECTORY path - Play is run from the constants.PUBLIC_CONFIGURATION_DIR - Play is run using the constants.ANSIBLE_CONFIG configuration file Args: pipeline (gomatic.Pipeline): playbook_with_path (str): play (str): deployment (str): edx_environment (str): app_repo (str) : private_github_key (str): hipchat_token (str): hipchat_room (str): manual_approval (bool): configuration_secure_dir (str): The secure config directory to use for this play. **kwargs (dict): k,v pairs: k: the name of the option to pass to ansible v: the value to use for this option Returns: gomatic.Stage """ # setup the necessary environment variables pipeline.ensure_encrypted_environment_variables({ 'HIPCHAT_TOKEN': hipchat_token, 'PRIVATE_GITHUB_KEY': private_github_key }) pipeline.ensure_environment_variables({ 'PLAY': play, 'DEPLOYMENT': deployment, 'EDX_ENVIRONMENT': edx_environment, 'APP_REPO': app_repo, 'ARTIFACT_PATH': '{}/'.format(constants.ARTIFACT_PATH), 'HIPCHAT_ROOM': hipchat_room, 'ANSIBLE_CONFIG': constants.ANSIBLE_CONTINUOUS_DELIVERY_CONFIG, }) stage = pipeline.ensure_stage(constants.RUN_PLAY_STAGE_NAME) if manual_approval: stage.set_has_manual_approval() # Install the requirements. job = stage.ensure_job(constants.RUN_PLAY_JOB_NAME) tasks.generate_requirements_install(job, 'tubular') tasks.generate_requirements_install(job, 'configuration') tasks.generate_target_directory(job) # fetch the key material artifact_params = { 'pipeline': pipeline.name, 'stage': constants.LAUNCH_INSTANCE_STAGE_NAME, 'job': constants.LAUNCH_INSTANCE_JOB_NAME, 'src': FetchArtifactFile("key.pem"), 'dest': constants.ARTIFACT_PATH } job.add_task(FetchArtifactTask(**artifact_params)) # fetch the launch_info.yml artifact_params['src'] = FetchArtifactFile('launch_info.yml') job.add_task(FetchArtifactTask(**artifact_params)) # fetch the inventory file artifact_params['src'] = FetchArtifactFile('ansible_inventory') job.add_task(FetchArtifactTask(**artifact_params)) tasks.generate_run_app_playbook(job, configuration_internal_dir, configuration_secure_dir, playbook_with_path, **kwargs) return stage
def generate_deploy_ami(stage, ami_artifact_location, edp, config, has_migrations=True, application_user=None, additional_migrations=None, management_commands=None,): """ Generates a job for deploying an AMI. Migrations are applied as part of this job. Args: stage (gomatic.gocd.pipelines.Stage): Stage to which this job belongs. ami_artifact_location (edxpipelines.utils.ArtifactLocation): Where to find the AMI artifact to deploy. edp (edxpipelines.utils.EDP): Tuple indicating environment, deployment, and play to which the AMI belongs. config (dict): Environment-specific secure config. has_migrations (bool): Whether to generate Gomatic for applying migrations. management_commands (list): management commands to run, will only run if list is defined application_user (str): application user if different from the play name. additional_migrations (list[edxpipelines.utils.MigrationAppInfo]): Additional applications to migrate. Will only run if has_migrations=True Returns: gomatic.gocd.pipelines.Job """ if not additional_migrations: additional_migrations = [] job = stage.ensure_job(constants.DEPLOY_AMI_JOB_NAME_TPL(edp)) tasks.generate_requirements_install(job, 'configuration') tasks.generate_package_install(job, 'tubular') tasks.generate_target_directory(job) # Retrieve the AMI ID from the upstream build stage. tasks.retrieve_artifact(ami_artifact_location, job) variable_override_path = path_to_artifact(ami_artifact_location.file_name) if has_migrations or management_commands: tasks.generate_launch_instance( job, aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], ec2_vpc_subnet_id=config['ec2_vpc_subnet_id'], ec2_security_group_id=config['ec2_security_group_id'], ec2_instance_profile_name=config['ec2_instance_profile_name'], variable_override_path=variable_override_path, ) # SSH key used to access the instance needs specific permissions. job.ensure_task(tasks.bash_task( 'chmod 600 {key_pem_path}', key_pem_path=path_to_artifact(constants.KEY_PEM_FILENAME) )) if application_user is None: application_user = edp.play if has_migrations: tasks.generate_run_migrations( job, application_user=application_user, application_name=application_user, application_path='/edx/app/{}'.format(application_user), db_migration_user=constants.DB_MIGRATION_USER, db_migration_pass=config['db_migration_pass'], ) for migration in additional_migrations: tasks.generate_run_migrations( job, application_user=migration.name, application_name=migration.name, application_path=migration.path, db_migration_user=constants.DB_MIGRATION_USER, db_migration_pass=config['db_migration_pass'], sub_application_name=migration.sub_application_name ) if management_commands: for command in management_commands: tasks.generate_run_management_command( job, application_user=application_user, application_name=application_user, application_path='/edx/app/{}'.format(application_user), command=command ) if has_migrations or management_commands: tasks.generate_ami_cleanup(job, config['slack_token'], runif='any') tasks.generate_deploy_ami( job, variable_override_path, config['asgard_api_endpoints'], config['asgard_token'], config['aws_access_key_id'], config['aws_secret_access_key'], ) return job
def generate_run_migrations(pipeline, db_migration_pass, inventory_location, instance_key_location, launch_info_location, application_user, application_name, application_path, sub_application_name=None, manual_approval=False): """ Generate the stage that applies/runs migrations. Args: pipeline (gomatic.Pipeline): Pipeline to which to add the run migrations stage. db_migration_pass (str): Password for the DB user used to run migrations. inventory_location (ArtifactLocation): Location of inventory containing the IP address of the EC2 instance, for fetching. instance_key_location (ArtifactLocation): Location of SSH key used to access the EC2 instance, for fetching. launch_info_location (ArtifactLocation): Location of the launch_info.yml file for fetching application_user (str): Username to use while running the migrations application_name (str): Name of the application (e.g. edxapp, programs, etc...) application_path (str): path of the application installed on the target machine sub_application_name (str): any sub application to insert in to the migrations commands {cms|lms} manual_approval (bool): Should this stage require manual approval? Returns: gomatic.Stage """ pipeline.ensure_environment_variables({ 'APPLICATION_USER': application_user, 'APPLICATION_NAME': application_name, 'APPLICATION_PATH': application_path, 'DB_MIGRATION_USER': '******', 'ARTIFACT_PATH': constants.ARTIFACT_PATH, 'ANSIBLE_CONFIG': constants.ANSIBLE_CONTINUOUS_DELIVERY_CONFIG, }) pipeline.ensure_encrypted_environment_variables({ 'DB_MIGRATION_PASS': db_migration_pass, }) if sub_application_name is not None: stage_name = "{}_{}".format(constants.APPLY_MIGRATIONS_STAGE, sub_application_name) else: stage_name = constants.APPLY_MIGRATIONS_STAGE stage = pipeline.ensure_stage(stage_name) if manual_approval: stage.set_has_manual_approval() job = stage.ensure_job(constants.APPLY_MIGRATIONS_JOB) # Fetch the Ansible inventory to use in reaching the EC2 instance. artifact_params = { "pipeline": inventory_location.pipeline, "stage": inventory_location.stage, "job": inventory_location.job, "src": FetchArtifactFile(inventory_location.file_name), "dest": constants.ARTIFACT_PATH } job.add_task(FetchArtifactTask(**artifact_params)) # Fetch the SSH key to use in reaching the EC2 instance. artifact_params = { "pipeline": instance_key_location.pipeline, "stage": instance_key_location.stage, "job": instance_key_location.job, "src": FetchArtifactFile(instance_key_location.file_name), "dest": constants.ARTIFACT_PATH } job.add_task(FetchArtifactTask(**artifact_params)) # ensure the target directoy exists tasks.generate_target_directory(job) # fetch the launch_info.yml artifact_params = { "pipeline": launch_info_location.pipeline, "stage": launch_info_location.stage, "job": launch_info_location.job, "src": FetchArtifactFile(launch_info_location.file_name), "dest": constants.ARTIFACT_PATH } job.add_task(FetchArtifactTask(**artifact_params)) # The SSH key used to access the EC2 instance needs specific permissions. job.add_task( ExecTask([ '/bin/bash', '-c', 'chmod 600 {}'.format( instance_key_location.file_name) ], working_dir=constants.ARTIFACT_PATH)) tasks.generate_requirements_install(job, 'configuration') tasks.generate_run_migrations(job, sub_application_name) # Cleanup EC2 instance if running the migrations failed. # I think this should be left for the terminate instance stage # tasks.generate_ami_cleanup(job, runif='failed') return stage
def generate_run_play(pipeline, playbook_with_path, edp, app_repo, slack_token='', slack_room=constants.SLACK_ROOM, manual_approval=False, configuration_secure_dir=constants.PRIVATE_CONFIGURATION_LOCAL_DIR, configuration_internal_dir=constants.INTERNAL_CONFIGURATION_LOCAL_DIR, override_artifacts=None, timeout=None, **kwargs): """ TODO: This currently runs from the configuration/playbooks/continuous_delivery/ directory. Need to figure out how to pass in a configuration file to ansible-play correctly. TE-1608 Assumes: - generate_launch_instance stage was used launch the instance preceding this stage. - Requires the ansible_inventory and key.pem files to be in the constants.ARTIFACT_DIRECTORY path - Play is run from the constants.PUBLIC_CONFIGURATION_DIR - Play is run using the constants.ANSIBLE_CONFIG configuration file Args: pipeline (gomatic.Pipeline): playbook_with_path (str): app_repo (str) : slack_token (str): Token used to authenticate to Slack. slack_room (str): Slack room to which to post notifications. manual_approval (bool): configuration_secure_dir (str): The secure config directory to use for this play. configuration_internal_dir (str): The internal config directory to use for this play. override_artifacts (bool): timeout (int): GoCD job level inactivity timeout setting. **kwargs (dict): k,v pairs: k: the name of the option to pass to ansible v: the value to use for this option Returns: gomatic.Stage """ stage = pipeline.ensure_stage(constants.RUN_PLAY_STAGE_NAME) if manual_approval: stage.set_has_manual_approval() # Install the requirements. job = stage.ensure_job(constants.RUN_PLAY_JOB_NAME) if timeout: job.timeout = str(timeout) tasks.generate_package_install(job, 'tubular') tasks.generate_requirements_install(job, 'configuration') tasks.generate_target_directory(job) for file_name in ( constants.KEY_PEM_FILENAME, constants.LAUNCH_INSTANCE_FILENAME, constants.ANSIBLE_INVENTORY_FILENAME ): tasks.retrieve_artifact( ArtifactLocation( pipeline.name, constants.LAUNCH_INSTANCE_STAGE_NAME, constants.LAUNCH_INSTANCE_JOB_NAME, file_name, ), job, constants.ARTIFACT_PATH ) override_files = [] if not override_artifacts: override_artifacts = [] for artifact in override_artifacts: tasks.retrieve_artifact(artifact, job, constants.ARTIFACT_PATH) override_files.append('{}/{}'.format(constants.ARTIFACT_PATH, artifact.file_name)) tasks.generate_run_app_playbook( job=job, playbook_with_path=playbook_with_path, edp=edp, app_repo=app_repo, slack_token=slack_token, slack_room=slack_room, configuration_secure_dir=configuration_secure_dir, configuration_internal_dir=configuration_internal_dir, override_files=override_files, **kwargs) return stage
def install_pipeline(save_config_locally, dry_run, variable_files, cmd_line_vars): config = utils.merge_files_and_dicts(variable_files, list(cmd_line_vars, )) configurator = GoCdConfigurator( HostRestClient(config['gocd_url'], config['gocd_username'], config['gocd_password'], ssl=True)) pipeline = configurator \ .ensure_pipeline_group(DRUPAL_PIPELINE_GROUP_NAME) \ .ensure_replacement_of_pipeline('rollback-prod-marketing-site') \ .set_git_material(GitMaterial('https://github.com/edx/tubular', polling=False, destination_directory='tubular', ignore_patterns=constants.MATERIAL_IGNORE_ALL_REGEX ) ) \ .ensure_material(PipelineMaterial(DEPLOY_MARKETING_PIPELINE_NAME, FETCH_TAG_STAGE_NAME)) pipeline.ensure_environment_variables({ 'MARKETING_REPOSITORY_VERSION': config['mktg_repository_version'], }) pipeline.ensure_encrypted_environment_variables({ 'PRIVATE_GITHUB_KEY': config['github_private_key'], 'PRIVATE_MARKETING_REPOSITORY_URL': config['mktg_repository_url'], 'PRIVATE_ACQUIA_USERNAME': config['acquia_username'], 'PRIVATE_ACQUIA_PASSWORD': config['acquia_password'], 'PRIVATE_ACQUIA_GITHUB_KEY': config['acquia_github_key'], }) prod_tag_name_artifact_params = { 'pipeline': DEPLOY_MARKETING_PIPELINE_NAME, 'stage': FETCH_TAG_STAGE_NAME, 'job': FETCH_TAG_JOB_NAME, 'src': FetchArtifactFile('{prod_tag}.txt'.format(prod_tag=PROD_TAG_NAME)), 'dest': 'target' } # Stage to rollback stage to its last stable tag rollback_stage = pipeline.ensure_stage(ROLLBACK_STAGE_NAME) rollback_stage.set_has_manual_approval() rollback_job = rollback_stage.ensure_job(ROLLBACK_JOB_NAME) tasks.generate_requirements_install(rollback_job, 'tubular') tasks.generate_target_directory(rollback_job) rollback_job.add_task(FetchArtifactTask(**prod_tag_name_artifact_params)) tasks.generate_drupal_deploy( rollback_job, PROD_ENV, '{prod_tag}.txt'.format(prod_tag=PROD_TAG_NAME)) # Stage to clear caches in extra clear_prod_caches_stage = pipeline.ensure_stage( CLEAR_PROD_CACHES_STAGE_NAME) clear_prod_caches_job = clear_prod_caches_stage.ensure_job( CLEAR_PROD_CACHES_JOB_NAME) tasks.fetch_edx_mktg(clear_prod_caches_job, 'edx-mktg') tasks.generate_requirements_install(clear_prod_caches_job, 'tubular') tasks.format_RSA_key(clear_prod_caches_job, 'edx-mktg/docroot/acquia_github_key.pem', '$PRIVATE_ACQUIA_GITHUB_KEY') tasks.generate_flush_drupal_caches(clear_prod_caches_job, PROD_ENV) tasks.generate_clear_varnish_cache(clear_prod_caches_job, PROD_ENV) configurator.save_updated_config(save_config_locally=save_config_locally, dry_run=dry_run)
def generate_deploy_ami(stage, ami_artifact_location, edp, config, has_migrations=True, application_user=None): """ Generates a job for deploying an AMI. Migrations are applied as part of this job. Args: stage (gomatic.gocd.pipelines.Stage): Stage to which this job belongs. ami_artifact_location (edxpipelines.utils.ArtifactLocation): Where to find the AMI artifact to deploy. edp (edxpipelines.utils.EDP): Tuple indicating environment, deployment, and play to which the AMI belongs. config (dict): Environment-specific secure config. has_migrations (bool): Whether to generate Gomatic for applying migrations. application_user (str): application user if different from the play name. Returns: gomatic.gocd.pipelines.Job """ job = stage.ensure_job(constants.DEPLOY_AMI_JOB_NAME_TPL(edp)) tasks.generate_requirements_install(job, 'configuration') tasks.generate_package_install(job, 'tubular') tasks.generate_target_directory(job) # Retrieve the AMI ID from the upstream build stage. tasks.retrieve_artifact(ami_artifact_location, job) variable_override_path = path_to_artifact(ami_artifact_location.file_name) if has_migrations: tasks.generate_launch_instance( job, aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], ec2_vpc_subnet_id=config['ec2_vpc_subnet_id'], ec2_security_group_id=config['ec2_security_group_id'], ec2_instance_profile_name=config['ec2_instance_profile_name'], variable_override_path=variable_override_path, ) # SSH key used to access the instance needs specific permissions. job.ensure_task(tasks.bash_task( 'chmod 600 {key_pem_path}', key_pem_path=path_to_artifact(constants.KEY_PEM_FILENAME) )) if application_user is None: application_user = edp.play tasks.generate_run_migrations( job, application_user=application_user, application_name=application_user, application_path='/edx/app/{}'.format(application_user), db_migration_user=constants.DB_MIGRATION_USER, db_migration_pass=config['db_migration_pass'], ) tasks.generate_ami_cleanup(job, config['hipchat_token'], runif='any') tasks.generate_deploy_ami( job, variable_override_path, config['asgard_api_endpoints'], config['asgard_token'], ) return job