def setUp(self): self.dagbag = models.DagBag(include_examples=True) self.dag1 = self.dagbag.dags['test_example_bash_operator'] self.dag2 = self.dagbag.dags['example_subdag_operator'] self.execution_dates = [days_ago(3), days_ago(2), days_ago(1)] self.session = Session()
def setUp(self): self.dagbag = models.DagBag(include_examples=True) self.dag1 = self.dagbag.dags['test_example_bash_operator'] self.dag2 = self.dagbag.dags['example_subdag_operator'] self.execution_dates = [days_ago(2), days_ago(1)] drs = _create_dagruns(self.dag1, self.execution_dates, state=State.RUNNING, run_id_template="scheduled__{}") for dr in drs: dr.dag = self.dag1 dr.verify_integrity() drs = _create_dagruns(self.dag2, [self.dag2.default_args['start_date']], state=State.RUNNING, run_id_template="scheduled__{}") for dr in drs: dr.dag = self.dag2 dr.verify_integrity() self.session = Session()
def test_days_ago(self): today = pendulum.today() today_midnight = pendulum.instance(datetime.fromordinal(today.date().toordinal())) self.assertTrue(dates.days_ago(0) == today_midnight) self.assertTrue( dates.days_ago(100) == today_midnight + timedelta(days=-100)) self.assertTrue( dates.days_ago(0, hour=3) == today_midnight + timedelta(hours=3)) self.assertTrue( dates.days_ago(0, minute=3) == today_midnight + timedelta(minutes=3)) self.assertTrue( dates.days_ago(0, second=3) == today_midnight + timedelta(seconds=3)) self.assertTrue( dates.days_ago(0, microsecond=3) == today_midnight + timedelta(microseconds=3))
# KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from xTool.utils.dates import days_ago from xTool.utils.log.logging_mixin import LoggingMixin from airflow.models import DAG log = LoggingMixin().log try: # Kubernetes is optional, so not available in vanilla Airflow # pip install apache-airflow[kubernetes] from airflow.contrib.operators.kubernetes_pod_operator import KubernetesPodOperator args = {'owner': 'airflow', 'start_date': days_ago(2)} dag = DAG(dag_id='example_kubernetes_operator', default_args=args, schedule_interval=None) k = KubernetesPodOperator(namespace='default', image="ubuntu:16.04", cmds=["bash", "-cx"], arguments=["echo", "10"], labels={"foo": "bar"}, name="airflow-test-pod", in_cluster=False, task_id="task", get_logs=True, dag=dag)
Run a shell command from Qubole Analyze against your Airflow cluster with following to trigger it manually `airflow trigger_dag example_qubole_sensor`. *Note: Make sure that connection `qubole_default` is properly set before running this example.* """ from airflow import DAG from airflow.contrib.sensors.qubole_sensor import QuboleFileSensor, QubolePartitionSensor from xTool.utils import dates default_args = { 'owner': 'airflow', 'depends_on_past': False, 'start_date': dates.days_ago(2), 'email': ['*****@*****.**'], 'email_on_failure': False, 'email_on_retry': False } dag = DAG('example_qubole_sensor', default_args=default_args, schedule_interval=None) dag.doc_md = __doc__ t1 = QuboleFileSensor( task_id='check_s3_file', qubole_conn_id='qubole_default', poke_interval=60, timeout=600, data={
class TestAirflowBaseViews(TestBase): EXAMPLE_DAG_DEFAULT_DATE = dates.days_ago(2) run_id = "test_{}".format( models.DagRun.id_for_date(EXAMPLE_DAG_DEFAULT_DATE)) def setUp(self): super(TestAirflowBaseViews, self).setUp() self.cleanup_dagruns() self.prepare_dagruns() def cleanup_dagruns(self): DR = models.DagRun dag_ids = [ 'example_bash_operator', 'example_subdag_operator', 'example_xcom' ] (self.session.query(DR).filter(DR.dag_id.in_(dag_ids)).filter( DR.run_id == self.run_id).delete(synchronize_session='fetch')) self.session.commit() def prepare_dagruns(self): dagbag = models.DagBag(include_examples=True) self.bash_dag = dagbag.dags['example_bash_operator'] self.sub_dag = dagbag.dags['example_subdag_operator'] self.xcom_dag = dagbag.dags['example_xcom'] self.bash_dagrun = self.bash_dag.create_dagrun( run_id=self.run_id, execution_date=self.EXAMPLE_DAG_DEFAULT_DATE, start_date=timezone.utcnow(), state=State.RUNNING) self.sub_dagrun = self.sub_dag.create_dagrun( run_id=self.run_id, execution_date=self.EXAMPLE_DAG_DEFAULT_DATE, start_date=timezone.utcnow(), state=State.RUNNING) self.xcom_dagrun = self.xcom_dag.create_dagrun( run_id=self.run_id, execution_date=self.EXAMPLE_DAG_DEFAULT_DATE, start_date=timezone.utcnow(), state=State.RUNNING) def test_index(self): resp = self.client.get('/', follow_redirects=True) self.check_content_in_response('DAGs', resp) def test_health(self): resp = self.client.get('health') self.check_content_in_response('The server is healthy!', resp) def test_home(self): resp = self.client.get('home', follow_redirects=True) self.check_content_in_response('DAGs', resp) def test_task(self): url = ( 'task?task_id=runme_0&dag_id=example_bash_operator&execution_date={}' .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE))) resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('Task Instance Details', resp) def test_xcom(self): url = ( 'xcom?task_id=runme_0&dag_id=example_bash_operator&execution_date={}' .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE))) resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('XCom', resp) def test_rendered(self): url = ( 'rendered?task_id=runme_0&dag_id=example_bash_operator&execution_date={}' .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE))) resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('Rendered Template', resp) def test_pickle_info(self): url = 'pickle_info?dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.assertEqual(resp.status_code, 200) def test_blocked(self): url = 'blocked' resp = self.client.get(url, follow_redirects=True) self.assertEqual(200, resp.status_code) def test_dag_stats(self): resp = self.client.get('dag_stats', follow_redirects=True) self.assertEqual(resp.status_code, 200) def test_task_stats(self): resp = self.client.get('task_stats', follow_redirects=True) self.assertEqual(resp.status_code, 200) def test_dag_details(self): url = 'dag_details?dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('DAG details', resp) def test_graph(self): url = 'graph?dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('runme_1', resp) def test_tree(self): url = 'tree?dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('runme_1', resp) def test_duration(self): url = 'duration?days=30&dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('example_bash_operator', resp) def test_tries(self): url = 'tries?days=30&dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('example_bash_operator', resp) def test_landing_times(self): url = 'landing_times?days=30&dag_id=test_example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('example_bash_operator', resp) def test_gantt(self): url = 'gantt?dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('example_bash_operator', resp) def test_code(self): url = 'code?dag_id=example_bash_operator' resp = self.client.get(url, follow_redirects=True) self.check_content_in_response('example_bash_operator', resp) def test_paused(self): url = 'paused?dag_id=example_bash_operator&is_paused=false' resp = self.client.post(url, follow_redirects=True) self.check_content_in_response('OK', resp) def test_success(self): url = ( 'success?task_id=run_this_last&dag_id=example_bash_operator&' 'execution_date={}&upstream=false&downstream=false&future=false&past=false' .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE))) resp = self.client.get(url) self.check_content_in_response('Wait a minute', resp) def test_clear(self): url = ( 'clear?task_id=runme_1&dag_id=example_bash_operator&' 'execution_date={}&upstream=false&downstream=false&future=false&past=false' .format(self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE))) resp = self.client.get(url) self.check_content_in_response( ['example_bash_operator', 'Wait a minute'], resp) def test_run(self): url = ( 'run?task_id=runme_0&dag_id=example_bash_operator&ignore_all_deps=false&' 'ignore_ti_state=true&execution_date={}'.format( self.percent_encode(self.EXAMPLE_DAG_DEFAULT_DATE))) resp = self.client.get(url) self.check_content_in_response('', resp, resp_code=302) def test_refresh(self): resp = self.client.get('refresh?dag_id=example_bash_operator') self.check_content_in_response('', resp, resp_code=302)