def test_partition_sensor(self, patched_poke):
patched_poke.return_value = True
sensor = QubolePartitionSensor(
task_id='test_qubole_partition_sensor',
data={
"schema": "default",
"table": "my_partitioned_table",
"columns": [{"column": "month", "values": ["1", "2"]}]
}
)
self.assertTrue(sensor.poke({}))
def test_partition_sensor(self, patched_poke):
patched_poke.return_value = True
sensor = QubolePartitionSensor(
task_id='test_qubole_partition_sensor',
data={
"schema": "default",
"table": "my_partitioned_table",
"columns": [{"column": "month", "values": ["1", "2"]}]
}
)
self.assertTrue(sensor.poke({}))
def test_partition_sensor_error(self, patched_poke):
patched_poke.return_value = True
dag = DAG(DAG_ID, start_date=DEFAULT_DATE)
with self.assertRaises(AirflowException):
QubolePartitionSensor(
task_id='test_qubole_partition_sensor',
poke_interval=1,
data={
"schema": "default",
"table": "my_partitioned_table",
"columns": [{"column": "month", "values": ["1", "2"]}]
},
dag=dag
)
t1 = QuboleFileSensor(
task_id='check_s3_file',
qubole_conn_id='qubole_default',
poke_interval=60,
timeout=600,
data={
"files":
[
"s3://paid-qubole/HadoopAPIExamples/jars/hadoop-0.20.1-dev-streaming.jar",
"s3://paid-qubole/HadoopAPITests/data/{{ ds.split('-')[2] }}.tsv"
] # will check for availability of all the files in array
}
)
t2 = QubolePartitionSensor(
task_id='check_hive_partition',
poke_interval=10,
timeout=60,
data={"schema": "default",
"table": "my_partitioned_table",
"columns": [
{"column": "month", "values":
["{{ ds.split('-')[1] }}"]},
{"column": "day", "values":
["{{ ds.split('-')[2] }}", "{{ yesterday_ds.split('-')[2] }}"]}
] # will check for partitions like [month=12/day=12,month=12/day=13]
}
)
t1 >> t2
