def setUp(self):
self.sensor = AthenaSensor(
task_id='test_athena_sensor',
query_execution_id='abc',
sleep_time=5,
max_retries=1,
aws_conn_id='aws_default',
)
class TestAthenaSensor(unittest.TestCase):
def setUp(self):
self.sensor = AthenaSensor(task_id='test_athena_sensor',
query_execution_id='abc',
sleep_time=5,
max_retires=1,
aws_conn_id='aws_default')
@mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("SUCCEEDED",))
def test_poke_success(self, mock_poll_query_status):
self.assertTrue(self.sensor.poke(None))
@mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("RUNNING",))
def test_poke_running(self, mock_poll_query_status):
self.assertFalse(self.sensor.poke(None))
@mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("QUEUED",))
def test_poke_queued(self, mock_poll_query_status):
self.assertFalse(self.sensor.poke(None))
@mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("FAILED",))
def test_poke_failed(self, mock_poll_query_status):
with self.assertRaises(AirflowException) as context:
self.sensor.poke(None)
self.assertIn('Athena sensor failed', str(context.exception))
@mock.patch.object(AWSAthenaHook, 'poll_query_status', side_effect=("CANCELLED",))
def test_poke_cancelled(self, mock_poll_query_status):
with self.assertRaises(AirflowException) as context:
self.sensor.poke(None)
self.assertIn('Athena sensor failed', str(context.exception))
class TestAthenaSensor(unittest.TestCase):
def setUp(self):
self.sensor = AthenaSensor(
task_id='test_athena_sensor',
query_execution_id='abc',
sleep_time=5,
max_retries=1,
aws_conn_id='aws_default',
)
@mock.patch.object(AWSAthenaHook,
'poll_query_status',
side_effect=("SUCCEEDED", ))
def test_poke_success(self, mock_poll_query_status):
assert self.sensor.poke(None)
@mock.patch.object(AWSAthenaHook,
'poll_query_status',
side_effect=("RUNNING", ))
def test_poke_running(self, mock_poll_query_status):
assert not self.sensor.poke(None)
@mock.patch.object(AWSAthenaHook,
'poll_query_status',
side_effect=("QUEUED", ))
def test_poke_queued(self, mock_poll_query_status):
assert not self.sensor.poke(None)
@mock.patch.object(AWSAthenaHook,
'poll_query_status',
side_effect=("FAILED", ))
def test_poke_failed(self, mock_poll_query_status):
with pytest.raises(AirflowException) as ctx:
self.sensor.poke(None)
assert 'Athena sensor failed' in str(ctx.value)
@mock.patch.object(AWSAthenaHook,
'poll_query_status',
side_effect=("CANCELLED", ))
def test_poke_cancelled(self, mock_poll_query_status):
with pytest.raises(AirflowException) as ctx:
self.sensor.poke(None)
assert 'Athena sensor failed' in str(ctx.value)
sleep_time=30,
max_tries=None,
)
read_table = AthenaOperator(
task_id='query__read_table',
query=QUERY_READ_TABLE,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
sleep_time=30,
max_tries=None,
)
get_read_state = AthenaSensor(
task_id='query__get_read_state',
query_execution_id=read_table.output,
max_retries=None,
sleep_time=10,
)
# Using a task-decorated function to read the results from S3
read_results_from_s3 = read_results_from_s3(read_table.output)
drop_table = AthenaOperator(
task_id='teardown__drop_table',
query=QUERY_DROP_TABLE,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
sleep_time=30,
max_tries=None,
)
)
read_table = AWSAthenaOperator(
task_id='query__read_table',
query=QUERY_READ_TABLE,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
sleep_time=30,
max_tries=None,
aws_conn_id=AWS_CONN_ID,
)
get_read_state = AthenaSensor(
task_id='query__get_read_state',
query_execution_id=
"{{ task_instance.xcom_pull('query__read_table', key='return_value') }}",
max_retries=None,
sleep_time=10,
aws_conn_id=AWS_CONN_ID,
)
# Using a task-decorated function to read the results from S3
read_results_from_s3 = read_results_from_s3(
"{{ task_instance.xcom_pull('query__read_table', key='return_value') }}"
)
drop_table = AWSAthenaOperator(
task_id='teardown__drop_table',
query=QUERY_DROP_TABLE,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
sleep_time=30,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
)
# [START howto_operator_athena]
read_table = AthenaOperator(
task_id='read_table',
query=QUERY_READ_TABLE,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
)
# [END howto_operator_athena]
# [START howto_sensor_athena]
await_query = AthenaSensor(
task_id='await_query',
query_execution_id=read_table.output,
)
# [END howto_sensor_athena]
drop_table = AthenaOperator(
task_id='drop_table',
query=QUERY_DROP_TABLE,
database=ATHENA_DATABASE,
output_location=f's3://{S3_BUCKET}/{S3_KEY}',
)
remove_s3_files = S3DeleteObjectsOperator(
task_id='remove_s3_files',
bucket=S3_BUCKET,
prefix=S3_KEY,
)