def setUp(self):
self.dagbag = models.DagBag(include_examples=True)
self.dag1 = self.dagbag.dags['test_example_bash_operator']
self.dag2 = self.dagbag.dags['example_subdag_operator']
self.execution_dates = [days_ago(3), days_ago(2), days_ago(1)]
self.session = Session()
def setUp(self):
self.dagbag = models.DagBag(include_examples=True)
self.dag1 = self.dagbag.dags['test_example_bash_operator']
self.dag2 = self.dagbag.dags['example_subdag_operator']
self.execution_dates = [days_ago(2), days_ago(1)]
drs = _create_dagruns(self.dag1, self.execution_dates,
state=State.RUNNING,
run_id_template="scheduled__{}")
for dr in drs:
dr.dag = self.dag1
dr.verify_integrity()
drs = _create_dagruns(self.dag2,
[self.dag2.default_args['start_date']],
state=State.RUNNING,
run_id_template="scheduled__{}")
for dr in drs:
dr.dag = self.dag2
dr.verify_integrity()
self.session = Session()
def test_days_ago(self):
today = pendulum.today()
today_midnight = pendulum.instance(datetime.fromordinal(today.date().toordinal()))
self.assertTrue(dates.days_ago(0) == today_midnight)
self.assertTrue(
dates.days_ago(100) == today_midnight + timedelta(days=-100))
self.assertTrue(
dates.days_ago(0, hour=3) == today_midnight + timedelta(hours=3))
self.assertTrue(
dates.days_ago(0, minute=3)
== today_midnight + timedelta(minutes=3))
self.assertTrue(
dates.days_ago(0, second=3)
== today_midnight + timedelta(seconds=3))
self.assertTrue(
dates.days_ago(0, microsecond=3)
== today_midnight + timedelta(microseconds=3))
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
from xTool.utils.dates import days_ago
from xTool.utils.log.logging_mixin import LoggingMixin
from airflow.models import DAG
log = LoggingMixin().log
try:
# Kubernetes is optional, so not available in vanilla Airflow
# pip install apache-airflow[kubernetes]
from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator
args = {'owner': 'airflow', 'start_date': days_ago(2)}
dag = DAG(dag_id='example_kubernetes_operator',
default_args=args,
schedule_interval=None)
k = KubernetesPodOperator(namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["echo", "10"],
labels={"foo": "bar"},
name="airflow-test-pod",
in_cluster=False,
task_id="task",
get_logs=True,
dag=dag)
Run a shell command from Qubole Analyze against your Airflow cluster with following to
trigger it manually `airflow trigger_dag example_qubole_sensor`.
*Note: Make sure that connection `qubole_default` is properly set before running
this example.*
"""
from airflow import DAG
from airflow.contrib.sensors.qubole_sensor import QuboleFileSensor, QubolePartitionSensor
from xTool.utils import dates
default_args = {
'owner': 'airflow',
'depends_on_past': False,
'start_date': dates.days_ago(2),
'email': ['*****@*****.**'],
'email_on_failure': False,
'email_on_retry': False
}
dag = DAG('example_qubole_sensor', default_args=default_args, schedule_interval=None)
dag.doc_md = __doc__
t1 = QuboleFileSensor(
task_id='check_s3_file',
qubole_conn_id='qubole_default',
poke_interval=60,
timeout=600,
data={
class TestAirflowBaseViews(TestBase):
EXAMPLE_DAG_DEFAULT_DATE = dates.days_ago(2)
run_id = "test_{}".format(
models.DagRun.id_for_date(EXAMPLE_DAG_DEFAULT_DATE))
def setUp(self):
super(TestAirflowBaseViews, self).setUp()
self.cleanup_dagruns()
self.prepare_dagruns()
def cleanup_dagruns(self):
DR = models.DagRun
dag_ids = [
'example_bash_operator', 'example_subdag_operator', 'example_xcom'
]
(self.session.query(DR).filter(DR.dag_id.in_(dag_ids)).filter(
DR.run_id == self.run_id).delete(synchronize_session='fetch'))
self.session.commit()
def prepare_dagruns(self):
dagbag = models.DagBag(include_examples=True)
self.bash_dag = dagbag.dags['example_bash_operator']
self.sub_dag = dagbag.dags['example_subdag_operator']
self.xcom_dag = dagbag.dags['example_xcom']
self.bash_dagrun = self.bash_dag.create_dagrun(
run_id=self.run_id,
execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
start_date=timezone.utcnow(),
state=State.RUNNING)
self.sub_dagrun = self.sub_dag.create_dagrun(
run_id=self.run_id,
execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
start_date=timezone.utcnow(),
state=State.RUNNING)
self.xcom_dagrun = self.xcom_dag.create_dagrun(
run_id=self.run_id,
execution_date=self.EXAMPLE_DAG_DEFAULT_DATE,
start_date=timezone.utcnow(),
state=State.RUNNING)
def test_index(self):
resp = self.client.get('/', follow_redirects=True)
self.check_content_in_response('DAGs', resp)
def test_health(self):
resp = self.client.get('health')
self.check_content_in_response('The server is healthy!', resp)
def test_home(self):
resp = self.client.get('home', follow_redirects=True)
self.check_content_in_response('DAGs', resp)
def test_task(self):
url = (
'task?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
.format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('Task Instance Details', resp)
def test_xcom(self):
url = (
'xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
.format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('XCom', resp)
def test_rendered(self):
url = (
'rendered?task_id=runme_0&dag_id=example_bash_operator&execution_date={}'
.format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('Rendered Template', resp)
def test_pickle_info(self):
url = 'pickle_info?dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.assertEqual(resp.status_code, 200)
def test_blocked(self):
url = 'blocked'
resp = self.client.get(url, follow_redirects=True)
self.assertEqual(200, resp.status_code)
def test_dag_stats(self):
resp = self.client.get('dag_stats', follow_redirects=True)
self.assertEqual(resp.status_code, 200)
def test_task_stats(self):
resp = self.client.get('task_stats', follow_redirects=True)
self.assertEqual(resp.status_code, 200)
def test_dag_details(self):
url = 'dag_details?dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('DAG details', resp)
def test_graph(self):
url = 'graph?dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('runme_1', resp)
def test_tree(self):
url = 'tree?dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('runme_1', resp)
def test_duration(self):
url = 'duration?days=30&dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('example_bash_operator', resp)
def test_tries(self):
url = 'tries?days=30&dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('example_bash_operator', resp)
def test_landing_times(self):
url = 'landing_times?days=30&dag_id=test_example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('example_bash_operator', resp)
def test_gantt(self):
url = 'gantt?dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('example_bash_operator', resp)
def test_code(self):
url = 'code?dag_id=example_bash_operator'
resp = self.client.get(url, follow_redirects=True)
self.check_content_in_response('example_bash_operator', resp)
def test_paused(self):
url = 'paused?dag_id=example_bash_operator&is_paused=false'
resp = self.client.post(url, follow_redirects=True)
self.check_content_in_response('OK', resp)
def test_success(self):
url = (
'success?task_id=run_this_last&dag_id=example_bash_operator&'
'execution_date={}&upstream=false&downstream=false&future=false&past=false'
.format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
resp = self.client.get(url)
self.check_content_in_response('Wait a minute', resp)
def test_clear(self):
url = (
'clear?task_id=runme_1&dag_id=example_bash_operator&'
'execution_date={}&upstream=false&downstream=false&future=false&past=false'
.format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
resp = self.client.get(url)
self.check_content_in_response(
['example_bash_operator', 'Wait a minute'], resp)
def test_run(self):
url = (
'run?task_id=runme_0&dag_id=example_bash_operator&ignore_all_deps=false&'
'ignore_ti_state=true&execution_date={}'.format(
self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE)))
resp = self.client.get(url)
self.check_content_in_response('', resp, resp_code=302)
def test_refresh(self):
resp = self.client.get('refresh?dag_id=example_bash_operator')
self.check_content_in_response('', resp, resp_code=302)