def test_partition_sensor(self, patched_poke): patched_poke.return_value = True sensor = QubolePartitionSensor( task_id='test_qubole_partition_sensor', data={ "schema": "default", "table": "my_partitioned_table", "columns": [{"column": "month", "values": ["1", "2"]}] } ) self.assertTrue(sensor.poke({}))
def test_partition_sensor_error(self, patched_poke): patched_poke.return_value = True dag = DAG(DAG_ID, start_date=DEFAULT_DATE) with self.assertRaises(AirflowException): QubolePartitionSensor( task_id='test_qubole_partition_sensor', poke_interval=1, data={ "schema": "default", "table": "my_partitioned_table", "columns": [{"column": "month", "values": ["1", "2"]}] }, dag=dag )
t1 = QuboleFileSensor( task_id='check_s3_file', qubole_conn_id='qubole_default', poke_interval=60, timeout=600, data={ "files": [ "s3://paid-qubole/HadoopAPIExamples/jars/hadoop-0.20.1-dev-streaming.jar", "s3://paid-qubole/HadoopAPITests/data/{{ ds.split('-')[2] }}.tsv" ] # will check for availability of all the files in array } ) t2 = QubolePartitionSensor( task_id='check_hive_partition', poke_interval=10, timeout=60, data={"schema": "default", "table": "my_partitioned_table", "columns": [ {"column": "month", "values": ["{{ ds.split('-')[1] }}"]}, {"column": "day", "values": ["{{ ds.split('-')[2] }}", "{{ yesterday_ds.split('-')[2] }}"]} ] # will check for partitions like [month=12/day=12,month=12/day=13] } ) t1 >> t2