}
dag = DAG('property_classification', default_args=default_args)
fetch_images_task = PythonOperator(task_id='fetch_images',
python_callable=fetch_images,
dag=dag,
op_kwargs={
'input_file': input_file_path,
'is_training': training
})
alex_net_task = PythonOperator(task_id='alex_net',
python_callable=alex,
dag=dag)
vgg_task = PythonOperator(task_id='vgg', python_callable=vgg, dag=dag)
res_net_task = PythonOperator(task_id='res_net',
python_callable=resnet,
dag=dag)
ensemble_task = PythonOperator(task_id='ensemble',
python_callable=final_predictions,
dag=dag)
# define the relationship between the tasks
fetch_images_task.set_downstream(alex_net_task)
alex_net_task.set_downstream(vgg_task)
vgg_task.set_downstream(res_net_task)
res_net_task.set_downstream(ensemble_task)
#todo: check run time for parallel execution of a,v,r
op_kwargs={
'wekeo_url':
BaseHook.get_connection("wekeo_hda").host,
'username':
BaseHook.get_connection("wekeo_hda").login,
'password':
BaseHook.get_connection("wekeo_hda").password,
'wekeo_job_id':
'EGUIBC37kepM90lTGVNTHpIdfuA',
'item_url':
'6a143583-a6a4-53e6-9e6d-8d4edc60a702/S5P_RPRO_L2__NO2____20180503T093059_20180503T111427_02866_01_010202_20190202T034117',
'output_filepath':
'./wekeo_data_storage/S5P_RPRO_L2__NO2____20180503T093059_20180503T111427_02866_01_010202_20190202T034117'
},
queue='process')
wekeo_0.set_downstream([load_collection_0])
wekeo_1 = PythonOperator(
task_id='wekeo_download_1',
dag=dag,
python_callable=download_wekeo_data,
op_kwargs={
'wekeo_url':
BaseHook.get_connection("wekeo_hda").host,
'username':
BaseHook.get_connection("wekeo_hda").login,
'password':
BaseHook.get_connection("wekeo_hda").password,
'wekeo_job_id':
'EGUIBC37kepM90lTGVNTHpIdfuA',
'item_url':
'email': ['*****@*****.**'],
'email_on_failure': True,
'email_on_retry': True,
'retries': 0,
}
with DAG(
'PriceAlerter',
default_args=default_args,
schedule_interval='*/5 9-17 * * *',
catchup=False,
) as dag:
price_listener = PythonOperator(
task_id='listener',
python_callable=listener.listen,
)
email_trigger = ShortCircuitOperator(
task_id='email_trigger',
python_callable=lambda: True if listener.trigger else False,
trigger_rule=TriggerRule.NONE_FAILED,
)
email = EmailOperator(
task_id='email',
to=email_service.email_list,
subject=email_service.get_email_subject(listener.summary),
html_content=email_service.get_html_content(listener.summary),
)
price_listener.set_downstream(email_trigger)
email_trigger.set_downstream(email)
t9 = \
PythonOperator(
task_id='Files_has_extracted',
python_callable=extract,
dag=dag)
t10 = \
PythonOperator(
task_id='Email_notify3',
python_callable=email,
dag=dag)
t1.set_downstream(t0)
t1.set_downstream(t2)
t2.set_downstream(t3)
t3.set_downstream(t0)
t3.set_downstream(t4)
t4.set_downstream(t5)
t5.set_downstream(t0)
t5.set_downstream(t6)
t6.set_downstream(t8)
t8.set_downstream(t9)
t8.set_downstream(t0)
t5.set_downstream(t7)
t7.set_downstream(t8)
t5.set_downstream(t8)
t9.set_downstream(t10)
t10.set_downstream(t0)
# replace user with your username
base_dir = '/home/user/idsp_pipeline'
default_args = {
'owner': 'user',
'depends_on_past': False,
'start_date': dt.datetime.strptime('2017-04-17T00:00:00',
'%Y-%m-%dT%H:%M:%S'),
'provide_context': True
}
dag = DAG('idsp_v1',
default_args=default_args,
schedule_interval='0 0 * * 2',
max_active_runs=1)
web_scrape_task = PythonOperator(task_id='scrape_web',
python_callable=scrape_web,
op_kwargs={'base_dir': base_dir},
dag=dag)
pdf_scrape_task = PythonOperator(task_id='scrape_pdf',
python_callable=scrape_pdf,
op_kwargs={'base_dir': base_dir},
dag=dag)
add_to_dataset_task = PythonOperator(task_id='add_to_dataset',
python_callable=add_to_dataset,
op_kwargs={'base_dir': base_dir},
dag=dag)
# define the relationship between the tasks using set_downstream
web_scrape_task.set_downstream(pdf_scrape_task)
pdf_scrape_task.set_downstream(add_to_dataset_task)
echo "{{ ds }}"
echo "{{ macros.ds_add(ds, 7)}}"
echo "{{ params.my_param }}"
{% endfor %}
"""
join = DummyOperator(
task_id='join',
trigger_rule='all_done',
dag=dag
)
sum_up = PythonOperator(
task_id='sum_up',
provide_context=True,
python_callable=sum_up_task,
dag=dag,
execution_timeout=timedelta(seconds=60),
on_failure_callback=notify_failure,
)
p1.set_upstream(p0)
p2.set_upstream(p0)
p3.set_upstream(p0)
c1.set_upstream(p1)
c2.set_upstream(p2)
c3.set_upstream(p3)
c3.set_downstream(join)
c2.set_downstream(join)
sum_up.set_upstream(join)
attachment.add_header("Content-Disposition", "attachment", filename=filename)
msg = MIMEMultipart()
msg.attach(attachment)
msg["Subject"] = str("Resultado Analise Notebook")
msg["From"] = "*****@*****.**"
msg["Reply-to"] = "*****@*****.**"
server = smtplib.SMTP("smtp.gmail.com:587")
server.ehlo()
server.starttls()
server.login("*****@*****.**", "a.24423242")
server.sendmail(msg["From"], "*****@*****.**", msg.as_string())
server.quit()
def acessNotebook():
print("Acessando Notebook no Swift")
runZero = PythonOperator(task_id="Acess_Notebook", provide_context=False, python_callable=acessNotebook, dag=dag)
runFirst = PythonOperator(task_id="ExecNotebook", provide_context=False, python_callable=execNotebook, dag=dag)
runSecond = PythonOperator(task_id="Get_Output", provide_context=False, python_callable=getOutput, dag=dag)
runThird = PythonOperator(task_id="send_email", provide_context=False, python_callable=send_email, dag=dag)
runZero.set_downstream(runFirst)
runFirst.set_downstream(runSecond)
runSecond.set_downstream(runThird)
wrtr.writerow(['url', 'count'])
wrtr.writerows(cntr.most_common(5))
simple_search = PythonOperator(task_id='search_twitter',
provide_context=True,
python_callable=search_twitter,
dag=dag,
params={'query': '#python'})
move_tweets_to_sqlite = PythonOperator(task_id='csv_to_sqlite',
provide_context=True,
python_callable=csv_to_sqlite,
dag=dag)
id_popular = PythonOperator(task_id='identify_popular_links',
provide_context=True,
python_callable=identify_popular_links,
dag=dag)
email_links = EmailOperator(task_id='email_best_links',
to='*****@*****.**',
subject='Latest popular links',
html_content='Check out the latest!!',
files=['{}/latest_links.txt'.format(RAW_TWEET_DIR)],
dag=dag)
simple_search.set_downstream(move_tweets_to_sqlite)
id_popular.set_upstream(move_tweets_to_sqlite)
email_links.set_upstream(id_popular)
html_content='Check out the latest!!',
files=['{}/latest_links.txt'.format(RAW_TWEET_DIR)],
dag=dag)
sub = SubDagOperator(subdag=subdag,
task_id='insert_and_id_pop',
trigger_rule='one_success',
dag=dag)
clear_latest = BashOperator(bash_command='rm -rf {}/latest_links.txt'.format(
RAW_TWEET_DIR), task_id='clear_latest', dag=dag)
gen_search_terms.set_upstream(fill_search_terms)
for term in SEARCH_TERMS:
term_without_punctuation = re.sub(r'\W+', '', term)
simple_search = PythonOperator(
task_id='search_{}_twitter'.format(term_without_punctuation),
provide_context=True,
python_callable=search_twitter,
dag=dag,
params={'query': term})
simple_search.set_upstream(gen_search_terms)
simple_search.set_downstream(sub)
sub.set_downstream(email_links)
email_links.set_downstream(clear_latest)
def create_dag(dag_id, schedule, start_date, delta_sensor, airpots_codes,
default_args):
dag = DAG(dag_id,
schedule_interval=schedule,
start_date=start_date,
default_args=default_args)
dag.doc_md = """
# DAG fetching data from smiles.com.ar
### procesing and dumping on postgresql
"""
"""start = TimeDeltaSensor(
task_id='wait_to_start',
delta=timedelta(minutes=delta_sensor),
dag=dag)"""
start = DummyOperator(task_id="start", dag=dag)
branches = []
def return_dates_branches(**kwargs):
return branches
gen_url_branch = BranchPythonOperator(
task_id='generate_url_dates',
provide_context=True,
python_callable=return_dates_branches,
dag=dag)
def transform_data(**kwargs):
ti = kwargs['ti']
raw_data = ti.xcom_pull(task_ids=return_dates_branches())
data = []
logging.info(raw_data)
if raw_data is not None:
flat_list = [item for sublist in raw_data for item in sublist]
for row in flat_list:
row = list(row)
# add À-ÿ for spanish accents
date = '/'.join(
list(
re.compile("([A-ZÀ-ÿ]+)(\d+)([A-ZÀ-ÿ]+)").split(
row[1]))[2:4])
date = dateparser.parse(date,
languages=['pt', 'es'],
date_formats=['%d/%b'
]).strftime('%Y-%m-%d')
row[1] = date
td = row[4].split(':')
row[4] = str(timedelta(hours=int(td[0]), minutes=int(td[1])))
row[5] = int(row[5].replace('.', ''))
row[6] = int(row[6].replace('.', ''))
row[8] = row[8].split(' ')[-1]
row.insert(0, datetime.now().strftime('%Y-%m-%d'))
data.append(tuple(row))
return data
else:
print('No se recibio datos')
t2 = PythonOperator(
task_id='transform_data',
python_callable=transform_data,
depends_on_past=True,
trigger_rule=TriggerRule.ALL_SUCCESS,
provide_context=True,
dag=dag,
)
t2.doc_md = """
#### Task Documentation
Transform fetched data
@return a list of tuples
"""
# def gen_url_dates(**kwargs):
date_start = read_scraped_date(airpots_codes)
date_end = date_start + timedelta(days=AMOUNT_DAYS)
date_generated = [
date_start + timedelta(days=x)
for x in range(0, (date_end - date_start).days)
]
for i, date in enumerate(date_generated):
date_ml = str(date.timestamp())[:8] + '00000'
url_dated = """https://www.smiles.com.ar/emission?originAirportCode={}&destinationAirportCode={}&departureDate={}&adults=1&children=0&infants=0&isFlexibleDateChecked=false&tripType=3¤cyCode=BRL&segments=2&departureDate2={}&originAirportCode2={}&destinationAirportCode2={}""".format(
airpots_codes[0][0], airpots_codes[1], date_ml, date_ml,
airpots_codes[0][1], airpots_codes[1])
get_data_op = PythonOperator(
task_id='get_data_{}and{}to{}_{}'.format(airpots_codes[0][0],
airpots_codes[0][1],
airpots_codes[1], i),
python_callable=get_data_URL,
op_kwargs={'URL': url_dated},
trigger_rule=TriggerRule.ONE_SUCCESS,
provide_context=True,
dag=dag,
)
branches.append(get_data_op.task_id)
get_data_op.set_upstream(gen_url_branch)
get_data_op.set_downstream(t2)
get_data_op.doc_md = """
#### Task Documentation
Fetch data from passed url
return list of semi-parsed data
"""
insert_data = PythonOperator(
task_id='insert_data',
python_callable=insert_into_table,
provide_context=True,
dag=dag,
)
insert_data.doc_md = """
#### Task Documentation
Insert parsed and transformed data into table
"""
t2.set_downstream(insert_data)
gen_url_branch.set_upstream(start)
return dag
subject='Latest popular links',
html_content='Check out the latest!!',
files=['{}/latest_links.txt'.format(RAW_TWEET_DIR)],
dag=dag)
sub = SubDagOperator(subdag=subdag,
task_id='insert_and_id_pop',
trigger_rule='one_success',
dag=dag)
clear_latest = BashOperator(
bash_command='rm -rf {}/latest_links.txt'.format(RAW_TWEET_DIR),
task_id='clear_latest',
dag=dag)
gen_search_terms.set_upstream(fill_search_terms)
for term in SEARCH_TERMS:
term_without_punctuation = re.sub(r'\W+', '', term)
simple_search = PythonOperator(
task_id='search_{}_twitter'.format(term_without_punctuation),
provide_context=True,
python_callable=search_twitter,
dag=dag,
params={'query': term})
simple_search.set_upstream(gen_search_terms)
simple_search.set_downstream(sub)
sub.set_downstream(email_links)
email_links.set_downstream(clear_latest)
entries_to_delete = session.query(airflow_db_model).filter(
age_check_column <= max_date,
).all()
logging.info("Process will be Deleting the following " + str(airflow_db_model.__name__) + "(s):")
for entry in entries_to_delete:
logging.info("\tEntry: " + str(entry) + ", Date: " + str(entry.__dict__[str(age_check_column).split(".")[1]]))
logging.info("Process will be Deleting " + str(len(entries_to_delete)) + " " + str(airflow_db_model.__name__) + "(s)")
if ENABLE_DELETE:
logging.info("Performing Delete...")
for entry in entries_to_delete:
session.delete(entry)
logging.info("Finished Performing Delete")
else:
logging.warn("You're opted to skip deleting the db entries!!!")
logging.info("Finished Running Cleanup Process")
for db_object in DATABASE_OBJECTS:
cleanup = PythonOperator(
task_id='cleanup_' + str(db_object["airflow_db_model"].__name__),
python_callable=cleanup_function,
params=db_object,
provide_context=True,
dag=dag
)
print_configuration.set_downstream(cleanup)
return 'Whatever you return gets printed in the logs'
setup_jobs = PythonOperator(task_id='setup_jobs',
provide_context=True,
python_callable=setup_jobs_fn,
dag=dag)
def collect_results_fn(ds, **kwargs):
pprint(kwargs)
print(ds)
collect_results = PythonOperator(task_id='collect_results',
provide_context=True,
python_callable=collect_results_fn,
dag=dag)
for i in range(10):
'''
Generating 10 sleeping task, sleeping from 0 to 9 seconds
respectively
'''
task = PythonOperator(task_id='sleep_for_' + str(i),
python_callable=my_sleeping_function,
op_kwargs={'random_base': float(i) / 10},
dag=dag)
task.set_upstream(setup_jobs)
task.set_downstream(collect_results)
t7 = PythonOperator(
task_id= 'vcf',
python_callable=create_container,
op_kwargs={'container_image':VCF_IMAGE, 'cmd': VCF_CMD},
provide_context=True,
dag=dag)
t8 = PythonOperator(
task_id= 'vcf_wait',
python_callable=noop,
op_kwargs={},
provide_context=True,
dag=dag)
t9 = PythonOperator(
task_id= 'vcf_cleanup',
python_callable=cleanup,
op_kwargs={},
provide_context=True,
dag=dag)
t1.set_downstream(t2)
t1.set_downstream(t3)
t2.set_downstream(t4)
t4.set_downstream(t5)
t4.set_downstream(t6)
t5.set_downstream(t7)
t7.set_downstream(t8)
t7.set_downstream(t9
simple_search = PythonOperator(task_id='search_twitter',
provide_context=True,
python_callable=search_twitter,
dag=dag,
params={'query': '#python'})
move_tweets_to_sqlite = PythonOperator(task_id='csv_to_sqlite',
provide_context=True,
python_callable=csv_to_sqlite,
dag=dag)
id_popular = PythonOperator(task_id='identify_popular_links',
provide_context=True,
python_callable=identify_popular_links,
dag=dag)
email_links = EmailOperator(task_id='email_best_links',
to='*****@*****.**',
subject='Latest popular links',
html_content='Check out the latest!!',
files=['{}/latest_links.txt'.format(RAW_TWEET_DIR)],
dag=dag)
simple_search.set_downstream(move_tweets_to_sqlite)
id_popular.set_upstream(move_tweets_to_sqlite)
email_links.set_upstream(id_popular)
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_retry': False,
'retries': 1,
'retry_delay': timedelta(minutes=1),
}
# DAG is scheduled to run every 8 hours
dag = DAG('PostTweet',
schedule_interval=timedelta(hours=8),
default_args=default_args)
# This dag will stage all the tweets to the csv file
t1 = PythonOperator(task_id='stage_tweets',
python_callable=stage_tweets,
dag=dag)
#This Dag will commit all the tweets to the csv file
t2 = PythonOperator(task_id='commit_tweets',
python_callable=commit_tweets,
dag=dag)
# This dag is used to send the tweets to twitter
t3 = PythonOperator(task_id='post_status', python_callable=post_tweet, dag=dag)
# Backup all the files and tweets to google drive
t4 = PythonOperator(task_id='backup', python_callable=upload.main, dag=dag)
t1.set_downstream(t2)
t2.set_downstream(t3)
t3.set_downstream(t4)
schedule_interval="@once")
# used to fatorize the code and avoid repetition
tabDays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
def get_day(**kwargs):
kwargs['ti'].xcom_push(key='day', value=datetime.now().weekday())
get_weekday = PythonOperator(
task_id='weekday',
python_callable=get_day,
provide_context=True,
dag=dag
)
def branch(**kwargs):
return 'task_for_' + 'wednesday'
fork = BranchPythonOperator(
task_id='branching',
python_callable=branch,
provide_context=True,
dag=dag)
get_weekday.set_downstream(fork)
for day in range(0, 6):
fork.set_downstream(DummyOperator(task_id='task_for_' + tabDays[day], dag=dag))
# Integrating read_data operator in airflow dag
read_table = PythonOperator(task_id='read_table',
python_callable=read_data,
op_kwargs={'fig_path': fig_path},
dag=dag)
# Integrating data_report operator in airflow dag
data_report = PythonOperator(task_id='data_report',
python_callable=data_report,
op_kwargs={'fig_path': fig_path},
dag=dag)
# Integrating plots operator in airflow dag
plots = PythonOperator(task_id='var_dist_plots',
python_callable=plot_var_distributions,
op_kwargs={'fig_path': fig_path},
dag=dag)
# Integrating train_test operator in airflow dag
train_test = PythonOperator(task_id='train_test',
python_callable=make_train_test,
op_kwargs={'fig_path': fig_path},
dag=dag)
# Integrating model_run operator in airflow dag
model_run = PythonOperator(task_id='model_run',
python_callable=run_model,
op_kwargs={'fig_path': fig_path},
dag=dag)
# Set the task sequence
read_table.set_downstream(data_report)
data_report.set_downstream([plots, train_test])
train_test.set_downstream(model_run)
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': airflow.utils.dates.days_ago(2),
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_retry': False
}
dag = DAG('athena_query_wk',
default_args=default_args,
dagrun_timeout=timedelta(hours=2),
schedule_interval='0 3 * * *')
submit_query = PythonOperator(task_id='submit_athena_query',
python_callable=run_athena_query,
op_kwargs={
'query': count_query,
'db': db_name,
's3_output': s3_ouput
},
dag=dag)
check_query_result = PythonOperator(task_id='check_query_result',
python_callable=check_query_status,
provide_context=True,
dag=dag)
submit_query.set_downstream(check_query_result)
task_id='setup_jobs',
provide_context=True,
python_callable=setup_jobs_fn,
dag=dag)
def collect_results_fn(ds, **kwargs):
pprint(kwargs)
print(ds)
collect_results = PythonOperator(
task_id='collect_results',
provide_context=True,
python_callable=collect_results_fn,
dag=dag)
for i in range(10):
'''
Generating 10 sleeping task, sleeping from 0 to 9 seconds
respectively
'''
task = PythonOperator(
task_id='sleep_for_'+str(i),
python_callable=my_sleeping_function,
op_kwargs={'random_base': float(i)/10},
dag=dag)
task.set_upstream(setup_jobs)
task.set_downstream(collect_results)
<table>
<tr><td><b> Task ID: </b></td><td>{{ task_instance.task_id }}</td></tr>
<tr><td><b> Execution Date: </b></td><td>{{ task_instance.execution_date }}</td></tr>
<tr><td><b> Start Date: </b></td><td>{{ task_instance.start_date }}</td></tr>
<tr><td><b> End Date: </b></td><td>{{ task_instance.end_date }}</td></tr>
<tr><td><b> Host Name: </b></td><td>{{ task_instance.hostname }}</td></tr>
<tr><td><b> Unix Name: </b></td><td>{{ task_instance.unixname }}</td></tr>
<tr><td><b> Job ID: </b></td><td>{{ task_instance.job_id }}</td></tr>
<tr><td><b> Queued Date Time: </b></td><td>{{ task_instance.queued_dttm }}</td></tr>
<tr><td><b> Log URL: </b></td><td><a href="{{ task_instance.log_url }}">{{ task_instance.log_url }}</a></td></tr>
</table>
<h2>Processes Killed</h2>
<ul>
{% for process_killed in task_instance.xcom_pull(task_ids='kill_halted_tasks', key='kill_halted_tasks.processes_to_kill') %}
<li>Process {{loop.index}}</li>
<ul>
{% for key, value in process_killed.iteritems() %}
<li>{{ key }}: {{ value }}</li>
{% endfor %}
</ul>
{% endfor %}
</ul>
</body>
</html>
""",
dag=dag)
kill_halted_tasks.set_downstream(email_or_not_branch)
email_or_not_branch.set_downstream(send_processes_killed_email)
'owner': 'ryan',
'depends_on_past': False,
'start_date': datetime.utcnow(),
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
# Run at the top of the hour Monday to Friday.
# Note: This doesn't line up with the market hours of
# 10PM Sunday till 10PM Friday GMT.
dag = DAG(dag_id='stocks',
default_args=args,
schedule_interval='0 * * * 1,2,3,4,5',
dagrun_timeout=timedelta(seconds=30))
# loop through the lob's we want to use to build up our dag
for stock in stocks:
get_stocks_task = \
PythonOperator(task_id='get_stocks',
provide_context=True,
op_kwargs={"stock": stock},
python_callable=get_stocks,
dag=dag)
cache_latest_stocks_task = \
PythonOperator(task_id='cache_latest_stocks',
provide_context=True,
python_callable=cache_latest_stocks,
dag=dag)
get_stocks_task.set_downstream(cache_latest_stocks_task)
args = {
'owner': 'mark',
'depends_on_past': False,
'start_date': datetime.utcnow(),
'retries': 1,
'retry_delay': timedelta(minutes=5),
}
# Run at the top of the hour Monday to Friday.
# Note: This doesn't line up with the market hours of
# 10PM Sunday till 10PM Friday GMT.
dag = DAG(dag_id='rates',
default_args=args,
schedule_interval='0 * * * 1,2,3,4,5',
dagrun_timeout=timedelta(seconds=30))
get_rates_task = \
PythonOperator(task_id='get_rates',
provide_context=True,
python_callable=get_rates,
dag=dag)
cache_latest_rates_task = \
PythonOperator(task_id='cache_latest_rates',
provide_context=True,
python_callable=cache_latest_rates,
dag=dag)
get_rates_task.set_downstream(cache_latest_rates_task)
dag7 = DAG(dag_id='test_subdag_deadlock', default_args=default_args)
subdag7 = DAG(dag_id='test_subdag_deadlock.subdag', default_args=default_args)
subdag7_task1 = PythonOperator(
task_id='test_subdag_fail',
dag=subdag7,
python_callable=fail)
subdag7_task2 = DummyOperator(
task_id='test_subdag_dummy_1',
dag=subdag7,)
subdag7_task3 = DummyOperator(
task_id='test_subdag_dummy_2',
dag=subdag7)
dag7_subdag1 = SubDagOperator(
task_id='subdag',
dag=dag7,
subdag=subdag7)
subdag7_task1.set_downstream(subdag7_task2)
subdag7_task2.set_downstream(subdag7_task3)
# DAG tests that queued tasks are run
dag8 = DAG(
dag_id='test_scheduled_queued_tasks',
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
default_args=default_args)
dag8_task1 = PythonOperator(
python_callable=fail,
task_id='test_queued_task',
dag=dag8,
pool='test_queued_pool')
task_id='test_depends_on_past_2',
depends_on_past=True,
dag=dag6,
)
dag6_task2.set_upstream(dag6_task1)
# DAG tests that a deadlocked subdag is properly caught
dag7 = DAG(dag_id='test_subdag_deadlock', default_args=default_args)
subdag7 = DAG(dag_id='test_subdag_deadlock.subdag', default_args=default_args)
subdag7_task1 = PythonOperator(task_id='test_subdag_fail',
dag=subdag7,
python_callable=fail)
subdag7_task2 = DummyOperator(
task_id='test_subdag_dummy_1',
dag=subdag7,
)
subdag7_task3 = DummyOperator(task_id='test_subdag_dummy_2', dag=subdag7)
dag7_subdag1 = SubDagOperator(task_id='subdag', dag=dag7, subdag=subdag7)
subdag7_task1.set_downstream(subdag7_task2)
subdag7_task2.set_downstream(subdag7_task3)
# DAG tests that queued tasks are run
dag8 = DAG(dag_id='test_scheduled_queued_tasks',
start_date=DEFAULT_DATE,
end_date=DEFAULT_DATE,
default_args=default_args)
dag8_task1 = PythonOperator(python_callable=fail,
task_id='test_queued_task',
dag=dag8,
pool='test_queued_pool')
dag=dag)
templated_command = """
{% for i in range(5) %}
echo "{{ ds }}"
echo "{{ macros.ds_add(ds, 7)}}"
echo "{{ params.my_param }}"
{% endfor %}
"""
join = DummyOperator(task_id='join', trigger_rule='all_done', dag=dag)
sum_up = PythonOperator(
task_id='sum_up',
provide_context=True,
python_callable=sum_up_task,
dag=dag,
execution_timeout=timedelta(seconds=60),
on_failure_callback=notify_failure,
)
p1.set_upstream(p0)
p2.set_upstream(p0)
p3.set_upstream(p0)
c1.set_upstream(p1)
c2.set_upstream(p2)
c3.set_upstream(p3)
c3.set_downstream(join)
c2.set_downstream(join)
sum_up.set_upstream(join)
