def test_execute_timeout(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
poke_interval=0, timeout=1)
mock_hook.return_value.pull.return_value = []
with self.assertRaises(AirflowSensorTimeout):
operator.execute(None)
mock_hook.return_value.pull.assert_called_with(
TEST_PROJECT, TEST_SUBSCRIPTION, 5, False)
def test_execute_timeout(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
poke_interval=0, timeout=1)
mock_hook.return_value.pull.return_value = []
with self.assertRaises(AirflowSensorTimeout):
operator.execute(None)
mock_hook.return_value.pull.assert_called_with(
TEST_PROJECT, TEST_SUBSCRIPTION, 5, False)
def test_execute(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
poke_interval=0)
generated_messages = self._generate_messages(5)
mock_hook.return_value.pull.return_value = generated_messages
response = operator.execute(None)
mock_hook.return_value.pull.assert_called_with(
TEST_PROJECT, TEST_SUBSCRIPTION, 5, False)
self.assertEquals(response, generated_messages)
def test_poke_with_ack_messages(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
ack_messages=True)
generated_messages = self._generate_messages(5)
mock_hook.return_value.pull.return_value = generated_messages
self.assertEquals(generated_messages, operator.poke(None))
mock_hook.return_value.acknowledge.assert_called_with(
TEST_PROJECT, TEST_SUBSCRIPTION, ['1', '2', '3', '4', '5']
)
def test_execute(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
poke_interval=0)
generated_messages = self._generate_messages(5)
mock_hook.return_value.pull.return_value = generated_messages
response = operator.execute(None)
mock_hook.return_value.pull.assert_called_with(
TEST_PROJECT, TEST_SUBSCRIPTION, 5, False)
self.assertEquals(response, generated_messages)
def test_poke_with_ack_messages(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION,
ack_messages=True)
generated_messages = self._generate_messages(5)
mock_hook.return_value.pull.return_value = generated_messages
self.assertEquals(generated_messages, operator.poke(None))
mock_hook.return_value.acknowledge.assert_called_with(
TEST_PROJECT, TEST_SUBSCRIPTION, ['1', '2', '3', '4', '5']
)
'email_on_failure': False,
'email_on_retry': False,
'project': project,
'topic': topic,
'subscription': subscription,
}
echo_template = '''
{% for m in task_instance.xcom_pull(task_ids='pull-messages') %}
echo "AckID: {{ m.get('ackId') }}, Base64-Encoded: {{ m.get('message') }}"
{% endfor %}
'''
with DAG('pubsub-end-to-end', default_args=default_args,
schedule_interval=datetime.timedelta(days=1)) as dag:
t1 = PubSubTopicCreateOperator(task_id='create-topic')
t2 = PubSubSubscriptionCreateOperator(
task_id='create-subscription', topic_project=project,
subscription=subscription)
t3 = PubSubPublishOperator(
task_id='publish-messages', messages=messages)
t4 = PubSubPullSensor(task_id='pull-messages', ack_messages=True)
t5 = BashOperator(task_id='echo-pulled-messages',
bash_command=echo_template)
t6 = PubSubSubscriptionDeleteOperator(task_id='delete-subscription')
t7 = PubSubTopicDeleteOperator(task_id='delete-topic')
t1 >> t2 >> t3
t2 >> t4 >> t5 >> t6 >> t7
def test_poke_no_messages(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION)
mock_hook.return_value.pull.return_value = []
self.assertEquals([], operator.poke(None))
def test_poke_no_messages(self, mock_hook):
operator = PubSubPullSensor(task_id=TASK_ID, project=TEST_PROJECT,
subscription=TEST_SUBSCRIPTION)
mock_hook.return_value.pull.return_value = []
self.assertEquals([], operator.poke(None))