from airflow import models
from airflow.operators import bash
# If you are running Airflow in more than one time zone
# see https://airflow.apache.org/docs/apache-airflow/stable/timezone.html
# for best practices
YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)
default_args = {
"owner": "Composer Example",
"depends_on_past": False,
"email": [""],
"email_on_failure": False,
"email_on_retry": False,
"retries": 1,
"retry_delay": datetime.timedelta(minutes=5),
"start_date": YESTERDAY,
}
with models.DAG(
"composer_sample_dag",
"catchup=False",
default_args=default_args,
schedule_interval=datetime.timedelta(days=1),
) as dag:
# Print the dag_run id from the Airflow logs
print_dag_run_conf = bash.BashOperator(
task_id="print_dag_run_conf", bash_command="echo {{ dag_run.id }}"
)
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""An example DAG demonstrating use of variables and how to test it."""
import datetime
from airflow import models
from airflow.operators import bash
from airflow.operators import dummy
yesterday = datetime.datetime.now() - datetime.timedelta(days=1)
default_dag_args = {
'start_date': yesterday,
}
with models.DAG('composer_sample_variables',
schedule_interval=datetime.timedelta(days=1),
default_args=default_dag_args) as dag:
start = dummy.DummyOperator(task_id='start')
end = dummy.DummyOperator(task_id='end')
variable_example = bash.BashOperator(task_id='variable_example',
bash_command='echo project_id=' +
models.Variable.get('gcp_project'))
start >> variable_example >> end
import datetime
import airflow
from airflow.operators import bash
# If you are running Airflow in more than one time zone
# see https://airflow.apache.org/docs/apache-airflow/stable/timezone.html
# for best practices
YESTERDAY = datetime.datetime.now() - datetime.timedelta(days=1)
default_args = {
'owner': 'Composer Example',
'depends_on_past': False,
'email': [''],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': datetime.timedelta(minutes=5),
'start_date': YESTERDAY,
}
with airflow.DAG('composer_sample_dag',
'catchup=False',
default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
# Print the dag_run id from the Airflow logs
print_dag_run_conf = bash.BashOperator(
task_id='print_dag_run_conf', bash_command='echo {{ dag_run.id }}')
# [END composer_quickstart]
'email_on_retry': False,
'retries': 1,
'retry_delay': datetime.timedelta(minutes=5),
'project_id': project_id
}
with models.DAG('composer_sample_bq_notify',
schedule_interval=datetime.timedelta(weeks=4),
default_args=default_dag_args) as dag:
# [END composer_notify_failure]
# [START composer_bash_bq]
# Create BigQuery output dataset.
make_bq_dataset = bash.BashOperator(
task_id='make_bq_dataset',
# Executing 'bq' command requires Google Cloud SDK which comes
# preinstalled in Cloud Composer.
bash_command=f'bq ls {bq_dataset_name} || bq mk {bq_dataset_name}')
# [END composer_bash_bq]
# [START composer_bigquery]
bq_recent_questions_query = bigquery.BigQueryInsertJobOperator(
task_id="bq_recent_questions_query",
configuration={
"query": {
"query": RECENT_QUESTIONS_QUERY,
"useLegacySql": False,
"destinationTable": {
"projectId": project_id,
"datasetId": bq_dataset_name,
"tableId": bq_recent_questions_table_id