AUTOML_MODEL = models.Variable.get('automl_model')
AUTOML_TRAINING_BUDGET = int(models.Variable.get('automl_training_budget'))
#[START dag_build_train_deploy]
default_dag_args = {
'start_date': datetime.datetime(2050, 1, 1),
'schedule_internal': None,
'provide_context': True
}
dag = models.DAG('build_train_deploy', default_args=default_dag_args)
#[END dag_build_train_deploy]
# instantiate Google Cloud base hook to get credentials and create automl clients
gcp_hook = GoogleCloudBaseHook(conn_id='google_cloud_default')
automl_client = AutoMlClient(credentials=gcp_hook._get_credentials())
# Loads the database dump from Cloud Storage to BigQuery
t1 = gcs_to_bq.GoogleCloudStorageToBigQueryOperator(
task_id="db_dump_to_bigquery",
bucket=COMPOSER_BUCKET_NAME,
source_objects=[DB_DUMP_FILENAME],
schema_object="schema_source.json",
source_format="CSV",
skip_leading_rows=1,
destination_project_dataset_table="{}.{}.{}".format(
PROJECT, DATASET, 'data_source'),
create_disposition="CREATE_IF_NEEDED",
write_disposition="WRITE_TRUNCATE",
dag=dag)
# this adds the error log url at the end of the msg
slack_msg = task_msg + """ (<{log_url}|log>)""".format(
log_url=context.get('task_instance').log_url, )
failed_alert = SlackWebhookOperator(
task_id='slack_test',
http_conn_id='slack',
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
)
return failed_alert.execute(context=context)
#to get credentials to access google sheets
wys_api_hook = GoogleCloudBaseHook('vz_api_google')
cred = wys_api_hook._get_credentials()
service = build('sheets', 'v4', credentials=cred, cache_discovery=False)
#to connect to pgadmin bot
wys_postgres = PostgresHook("wys_bot")
connection = BaseHook.get_connection('wys_api_key')
api_key = connection.password
default_args = {
'owner': 'rdumas',
'depends_on_past': False,
'start_date': datetime(2020, 4, 1),
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_success': False,
'retries': 0,
webhook_token=slack_webhook_token,
message=slack_msg,
username='******',
)
return failed_alert.execute(context=context)
#to read the python script for pulling data from google sheet and putting it into tables in postgres
try:
sys.path.append('/etc/airflow/data_scripts/vision_zero/')
from schools import pull_from_sheet
except:
raise ImportError("Cannot import functions to pull school safety zone list")
#to get credentials to access google sheets
vz_api_hook = GoogleCloudBaseHook('vz_api_google')
cred = vz_api_hook._get_credentials()
service = build('sheets', 'v4', credentials=cred, cache_discovery=False)
#To connect to pgadmin bot
vz_api_bot = PostgresHook("vz_api_bot")
con = vz_api_bot.get_conn()
DEFAULT_ARGS = {
'owner': 'cnangini',
'depends_on_past' : False,
'email': ['*****@*****.**'],
'email_on_failure': True,
'email_on_retry': True,
'start_date': datetime(2019, 9, 30),
'retries': 0,
'retry_delay': timedelta(minutes=5),