def DailyPipeline(branch):
def DailyGenerateTestArgs(**kwargs):
"""Loads the configuration that will be used for this Iteration."""
conf = kwargs['dag_run'].conf
if conf is None:
conf = dict()
# If variables are overridden then we should use it otherwise we use it's
# default value.
date = datetime.datetime.now()
date_string = date.strftime('%Y%m%d-%H-%M')
version = conf.get('VERSION')
if version is None:
# VERSION is of the form '{branch}-{date_string}'
version = '%s-%s' % (branch, date_string)
gcs_path = conf.get('GCS_DAILY_PATH')
if gcs_path is None:
# GCS_DAILY_PATH is of the form 'daily-build/{version}'
gcs_path = 'daily-build/%s' % (version)
commit = conf.get('COMMIT') or ""
mfest_commit = conf.get('MFEST_COMMIT')
if mfest_commit is None:
timestamp = time.mktime(date.timetuple())
# MFEST_COMMIT is of the form '{branch}@{{{timestamp}}}',
mfest_commit = '%s@{%s}' % (branch, timestamp)
default_conf = environment_config.GetDefaultAirflowConfig(
branch=branch,
commit=commit,
gcs_path=gcs_path,
mfest_commit=mfest_commit,
pipeline_type='daily',
verify_consistency='false',
version=version)
config_settings = dict()
for name in default_conf.iterkeys():
config_settings[name] = conf.get(name) or default_conf[name]
testDailyConfigSettings(config_settings)
return config_settings
dag_name = 'istio_daily_' + branch
dag, tasks, addAirflowBashOperator = istio_common_dag.MakeCommonDag(
DailyGenerateTestArgs,
name=dag_name, schedule_interval='15 9 * * *')
tasks['mark_daily_complete'] = MakeMarkComplete(dag, addAirflowBashOperator)
#tasks['generate_workflow_args']
tasks['get_git_commit' ].set_upstream(tasks['generate_workflow_args'])
tasks['run_cloud_builder' ].set_upstream(tasks['get_git_commit'])
tasks['modify_values_helm' ].set_upstream(tasks['run_cloud_builder'])
tasks['run_release_qualification_tests'].set_upstream(tasks['modify_values_helm'])
tasks['copy_files_for_release' ].set_upstream(tasks['run_release_qualification_tests'])
tasks['mark_daily_complete' ].set_upstream(tasks['copy_files_for_release'])
tasks['tag_daily_gcr' ].set_upstream(tasks['mark_daily_complete'])
return dag
See the License for the specific language governing permissions and
limitations under the License.
"""
import logging
import re
from airflow import DAG
from airflow.models import Variable
from airflow.operators.bash_operator import BashOperator
from airflow.operators.python_operator import PythonOperator
import environment_config
import istio_common_dag
dag, copy_files = istio_common_dag.MakeCommonDag(
'istio_monthly_release',
schedule_interval=environment_config.MONTHLY_RELEASE_TRIGGER,
monthly=True)
monthly_release_template = """
{% set m_commit = task_instance.xcom_pull(task_ids='get_git_commit') %}
{% set settings = task_instance.xcom_pull(task_ids='generate_workflow_args') %}
gsutil cp gs://{{ settings.GCS_RELEASE_TOOLS_PATH }}/data/release/*.json .
gsutil cp gs://{{ settings.GCS_RELEASE_TOOLS_PATH }}/data/release/*.sh .
chmod u+x *
./start_gcb_publish.sh \
-p "{{ settings.RELEASE_PROJECT_ID }}" -a "{{ settings.SVC_ACCT }}" \
-v "{{ settings.VERSION }}" -s "{{ settings.GCS_FULL_STAGING_PATH }}" \
-b "{{ settings.GCS_MONTHLY_RELEASE_PATH }}" -r "{{ settings.GCR_RELEASE_DEST }}" \
-g "{{ settings.GCS_GITHUB_PATH }}" -u "{{ settings.MFEST_URL }}" \
-t "{{ m_commit }}" -m "{{ settings.MFEST_FILE }}" \
-h "{{ settings.GITHUB_ORG }}" -i "{{ settings.GITHUB_REPO }}" \
def MonthlyPipeline():
MONTHLY_RELEASE_TRIGGER = '15 17 * * 4#3'
def MonthlyGenerateTestArgs(**kwargs):
"""Loads the configuration that will be used for this Iteration."""
conf = kwargs['dag_run'].conf
if conf is None:
conf = dict()
# If version is overridden then we should use it otherwise we use it's
# default or monthly value.
version = conf.get('VERSION') or istio_common_dag.GetVariableOrDefault(
'monthly-version', None)
if not version or version == 'INVALID':
raise ValueError('version needs to be provided')
Variable.set('monthly-version', 'INVALID')
#GCS_MONTHLY_STAGE_PATH is of the form ='prerelease/{version}'
gcs_path = 'prerelease/%s' % (version)
branch = conf.get('BRANCH') or istio_common_dag.GetVariableOrDefault(
'monthly-branch', None)
if not branch or branch == 'INVALID':
raise ValueError('branch needs to be provided')
Variable.set('monthly-branch', 'INVALID')
mfest_commit = conf.get('MFEST_COMMIT') or branch
default_conf = environment_config.GetDefaultAirflowConfig(
branch=branch,
gcs_path=gcs_path,
mfest_commit=mfest_commit,
pipeline_type='monthly',
verify_consistency='true',
version=version)
config_settings = dict()
for name in default_conf.iterkeys():
config_settings[name] = conf.get(name) or default_conf[name]
# These are the extra params that are passed to the dags for monthly release
monthly_conf = dict()
monthly_conf['DOCKER_HUB'] = 'istio'
monthly_conf['GCR_RELEASE_DEST'] = 'istio-io'
monthly_conf['GCS_GITHUB_PATH'] = 'istio-secrets/github.txt.enc'
monthly_conf['RELEASE_PROJECT_ID'] = 'istio-io'
# GCS_MONTHLY_RELEASE_PATH is of the form 'istio-release/releases/{version}'
monthly_conf[
'GCS_MONTHLY_RELEASE_PATH'] = 'istio-release/releases/%s' % (
version)
for name in monthly_conf.iterkeys():
config_settings[name] = conf.get(name) or monthly_conf[name]
testMonthlyConfigSettings(config_settings)
return config_settings
def ReportMonthlySuccessful(task_instance, **kwargs):
del kwargs
dag, tasks, addAirflowBashOperator = istio_common_dag.MakeCommonDag(
MonthlyGenerateTestArgs,
'istio_monthly_dag',
schedule_interval=MONTHLY_RELEASE_TRIGGER,
extra_param_lst=monthly_extra_params)
addAirflowBashOperator('release_push_github_docker_template',
'github_and_docker_release',
need_commit=True)
addAirflowBashOperator('release_tag_github_template',
'github_tag_repos',
need_commit=True)
mark_monthly_complete = PythonOperator(
task_id='mark_monthly_complete',
python_callable=ReportMonthlySuccessful,
provide_context=True,
dag=dag,
)
tasks['mark_monthly_complete'] = mark_monthly_complete
# tasks['generate_workflow_args']
tasks['get_git_commit'].set_upstream(tasks['generate_workflow_args'])
tasks['run_cloud_builder'].set_upstream(tasks['get_git_commit'])
tasks['run_release_qualification_tests'].set_upstream(
tasks['run_cloud_builder'])
tasks['modify_values_helm'].set_upstream(
tasks['run_release_qualification_tests'])
tasks['copy_files_for_release'].set_upstream(tasks['modify_values_helm'])
tasks['github_and_docker_release'].set_upstream(
tasks['copy_files_for_release'])
tasks['github_tag_repos'].set_upstream(tasks['github_and_docker_release'])
tasks['mark_monthly_complete'].set_upstream(tasks['github_tag_repos'])
return dag
def DailyPipeline(branch):
def DailyGenerateTestArgs(**kwargs):
"""Loads the configuration that will be used for this Iteration."""
env_conf = kwargs['dag_run'].conf
if env_conf is None:
env_conf = dict()
# If variables are overridden then we should use it otherwise we use it's
# default value.
date = datetime.datetime.now()
date_string = date.strftime('%Y%m%d-%H-%M')
docker_hub = env_conf.get('CB_DOCKER_HUB')
if docker_hub is None:
docker_hub = 'gcr.io/istio-release'
version = env_conf.get('CB_VERSION')
if version is None:
# VERSION is of the form '{branch}-{date_string}'
version = '%s-%s' % (branch, date_string)
gcs_path = env_conf.get('CB_GCS_DAILY_PATH')
if gcs_path is None:
# GCS_DAILY_PATH is of the form 'daily-build/{version}'
gcs_path = 'daily-build/%s' % (version)
commit = env_conf.get('CB_COMMIT') or ""
github_org = env_conf.get('CB_GITHUB_ORG') or "istio"
default_conf = environment_config.GetDefaultAirflowConfig(
branch=branch,
commit=commit,
docker_hub=docker_hub,
gcs_path=gcs_path,
github_org=github_org,
pipeline_type='daily',
verify_consistency='false',
version=version)
config_settings = istio_common_dag.MergeEnvironmentIntoConfig(
env_conf, default_conf)
testDailyConfigSettings(config_settings)
return config_settings
dag_name = 'istio_daily_' + branch
dag, tasks, addAirflowBashOperator = istio_common_dag.MakeCommonDag(
DailyGenerateTestArgs, name=dag_name, schedule_interval='15 9 * * *')
addAirflowBashOperator('gcr_tag_success', 'tag_daily_gcr')
#tasks['generate_workflow_args']
tasks['init_gcb_env'].set_upstream(tasks['generate_workflow_args'])
tasks['get_git_commit'].set_upstream(tasks['init_gcb_env'])
tasks['run_cloud_builder'].set_upstream(tasks['get_git_commit'])
tasks['modify_values_helm'].set_upstream(tasks['run_cloud_builder'])
tasks['run_release_qualification_tests'].set_upstream(
tasks['modify_values_helm'])
tasks['copy_files_for_release'].set_upstream(
tasks['run_release_qualification_tests'])
tasks['tag_daily_gcr'].set_upstream(tasks['copy_files_for_release'])
return dag
"""Airfow DAG used is the daily release pipeline.
Copyright 2017 Istio Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
from airflow import DAG
import istio_common_dag
dag, copy_files = istio_common_dag.MakeCommonDag(
name='istio_daily_release', schedule_interval='15 9 * * *')
mark_complete = istio_common_dag.MakeMarkComplete(dag)
copy_files >> mark_complete
dag
def MonthlyPipeline():
MONTHLY_RELEASE_TRIGGER = '15 17 * * 4#3'
def MonthlyGenerateTestArgs(**kwargs):
"""Loads the configuration that will be used for this Iteration."""
env_conf = kwargs['dag_run'].conf
if env_conf is None:
env_conf = dict()
docker_hub = env_conf.get('CB_DOCKER_HUB')
if docker_hub is None:
docker_hub = 'docker.io/istio'
# If version is overridden then we should use it otherwise we use it's
# default or monthly value.
version = env_conf.get(
'CB_VERSION') or istio_common_dag.GetVariableOrDefault(
'monthly-version', None)
if not version or version == 'INVALID':
raise ValueError('version needs to be provided')
Variable.set('monthly-version', 'INVALID')
#GCS_MONTHLY_STAGE_PATH is of the form ='prerelease/{version}'
gcs_path = 'prerelease/%s' % (version)
branch = env_conf.get(
'CB_BRANCH') or istio_common_dag.GetVariableOrDefault(
'monthly-branch', None)
if not branch or branch == 'INVALID':
raise ValueError('branch needs to be provided')
Variable.set('monthly-branch', 'INVALID')
commit = env_conf.get('CB_COMMIT') or branch
github_org = env_conf.get('CB_GITHUB_ORG') or "istio"
default_conf = environment_config.GetDefaultAirflowConfig(
branch=branch,
commit=commit,
docker_hub=docker_hub,
gcs_path=gcs_path,
github_org=github_org,
pipeline_type='monthly',
verify_consistency='true',
version=version)
# These are the extra params that are passed to the dags for monthly release
monthly_conf = dict()
# CB_GCS_MONTHLY_RELEASE_PATH is of the form 'istio-release/releases/{version}'
monthly_conf[
'CB_GCS_MONTHLY_RELEASE_PATH'] = 'istio-release/releases/%s' % (
version)
config_settings = istio_common_dag.MergeEnvironmentIntoConfig(
env_conf, default_conf, monthly_conf)
testMonthlyConfigSettings(config_settings)
return config_settings
dag, tasks, addAirflowBashOperator = istio_common_dag.MakeCommonDag(
MonthlyGenerateTestArgs,
'istio_monthly_dag',
schedule_interval=MONTHLY_RELEASE_TRIGGER,
extra_param_lst=monthly_extra_params)
addAirflowBashOperator('release_push_github_docker_template',
'github_and_docker_release')
addAirflowBashOperator('release_tag_github_template', 'github_tag_repos')
# tasks['generate_workflow_args']
tasks['init_gcb_env'].set_upstream(tasks['generate_workflow_args'])
tasks['get_git_commit'].set_upstream(tasks['init_gcb_env'])
tasks['run_cloud_builder'].set_upstream(tasks['get_git_commit'])
tasks['modify_values_helm'].set_upstream(tasks['run_cloud_builder'])
tasks['run_release_qualification_tests'].set_upstream(
tasks['modify_values_helm'])
tasks['copy_files_for_release'].set_upstream(
tasks['run_release_qualification_tests'])
tasks['github_and_docker_release'].set_upstream(
tasks['copy_files_for_release'])
tasks['github_tag_repos'].set_upstream(tasks['github_and_docker_release'])
return dag