def test_subscriptions(client, app, subscriptions_repo, callback_server, url,
topic, pattern):
topic = topic.format(TOPIC_BASE_URL=f'{app.config.TOPIC_BASE_URL}/')
subscription_callback = callback_server.valid_callback_url(1)
data = {
'hub.mode': 'subscribe',
'hub.topic': topic,
'hub.lease_seconds': 3600,
'hub.callback': subscription_callback
}
# creating subscription
# checking that the subscription does not exist
assert not subscriptions_repo.get_subscriptions_by_pattern(
Pattern(pattern))
response = client.post(url,
data=data,
mimetype='application/x-www-form-urlencoded')
assert response.status_code == HTTPStatus.OK, response.json
# checking that a single subscription was created for the expected pattern
assert len(
subscriptions_repo.get_subscriptions_by_pattern(Pattern(pattern))) == 1
# deleting subcription
data['hub.mode'] = 'unsubscribe'
response = client.post(url,
data=data,
mimetype='application/x-www-form-urlencoded')
assert response.status_code == HTTPStatus.OK, response.json
# subscription can't be deleted if it does not exist
response = client.post(url,
data=data,
mimetype='application/x-www-form-urlencoded')
assert response.status_code == HTTPStatus.NOT_FOUND, response.json
def execute(self, url, topic):
pattern = Pattern(topic)
subscriptions = self.subscriptions_repo.get_subscriptions_by_pattern(pattern)
subscriptions_by_url = [s for s in subscriptions if s.callback_url == url]
if not subscriptions_by_url:
raise SubscriptionNotFound()
self.subscriptions_repo.bulk_delete([pattern.to_key(url)])
def execute(self, callback: str = None, topic: str = None):
pattern = Pattern(topic)
subscriptions = self.subscriptions_repo.get_subscriptions_by_pattern(
pattern)
subscriptions_by_callbacks = [
s for s in subscriptions if s.callback_url == callback
]
if not subscriptions_by_callbacks:
raise SubscriptionNotFoundError()
self.subscriptions_repo.bulk_delete([pattern.to_key(callback)])
def test_get_subscriptions_by_pattern__should_return_subscriptions(self):
repo = SubscriptionsRepo(connection_data=self.connection_data)
self.client.list_objects.side_effect = [{
'Contents': []
}, {
'Contents': [{
'Key': 'AA/BB/ff0d1111f6636c354cf92c7137f1b5e6'
}]
}]
self.client.get_object.return_value = {
'Body':
BytesIO(
b'{"c": "http://callback.url/1", "e": "2020-05-18T15:08:00"}'),
'Bucket':
'subscriptions',
'ContentLength':
39,
'Key':
'AA',
}
subscriptions = repo.get_subscriptions_by_pattern(Pattern('aa.bb'))
assert len(list(subscriptions)) == 1
assert list(subscriptions)[0].callback_url == 'http://callback.url/1'
assert self.client.list_objects.mock_calls == [
mock.call(Bucket='subscriptions', Prefix='AA/', Delimiter='/'),
mock.call(Bucket='subscriptions', Prefix='AA/BB/', Delimiter='/')
]
self.client.get_object.assert_called_once_with(
Bucket='subscriptions',
Key='AA/BB/ff0d1111f6636c354cf92c7137f1b5e6')
def execute(self, topic: str = None, topic_prefix: str = None):
parsed_topic_url = urllib.parse.urlparse(topic)
if parsed_topic_url.scheme:
topic_canonical_url = topic
if topic_canonical_url.startswith(self.topic_base_url):
topic = topic[len(self.topic_base_url):]
if topic_prefix:
topic = f'{topic_prefix}.{topic}'
topic_canonical_url = urllib.parse.urljoin(
self.topic_base_url, topic)
try:
Pattern(topic)._validate()
except ValueError as e:
raise BadParametersError(
detail=f'"{topic}" is invalid topic string') from e
else:
raise BadParametersError(
detail=
f'Topic url "{topic_canonical_url}" must start with "{self.topic_base_url}"'
)
response = requests.get(topic_canonical_url)
if response.status_code == HTTPStatus.OK:
topic_response = response.json()
if topic_response == topic:
return topic
else:
raise ConflictError(
detail=
'Unexpected topic string returned by the channel, expected: "{}", got:"{}"'
.format(topic, topic_response))
elif response.status_code == HTTPStatus.NOT_FOUND:
raise NotFoundError(detail=f'Topic "{topic}" does not exist')
else:
raise UnexpectedTopicURLResponseError(
detail='Unexpeced response code {} from {}'.format(
response.status_code, topic_canonical_url))
else:
if topic_prefix:
topic = f'{topic_prefix}.{topic}'
try:
Pattern(topic)._validate()
return topic
except ValueError as e:
raise BadParametersError(
detail=f'"{topic}" is invalid topic string') from e
def execute(self, url, predicate, expiration=None):
# this operation deletes all previous subscription for given url and pattern
# and replaces them with new one. Techically it's create or update operation
super().execute()
posted = self.subscriptions_repo.subscribe_by_pattern(Pattern(predicate), url, expiration)
if not posted:
return None
return True
def test_subscribe_by_pattern__with_missing_expiration__should_return_error(
self):
repo = SubscriptionsRepo(connection_data=self.connection_data)
pattern = Pattern('aaa.bbb.ccc')
with self.assertRaises(SubscriptionMissingExpiration):
repo.subscribe_by_pattern(pattern,
'http://callback.url/1',
expiration_seconds=0)
def _fill_subscriptions_repo(repo, subscriptions):
overall = 0
for predicate, number in subscriptions.items():
overall += number
url_tail = predicate.replace('.', '-')+"-{}"
url = CALLBACK_URL.format(url_tail)
for i in range(number):
print(f"subcribe predicate:{predicate}, url:{url.format(i)}")
repo.subscribe_by_pattern(Pattern(predicate), url.format(i), DEFAULT_EXPIRATION)
assert not repo.is_empty()
def test_post__with_unsubscribe_mode__should_unsubscribe(self):
self.subscriptions_repo.subscribe_by_pattern(Pattern('id'),
'https://callback.url/1',
30)
assert self.subscriptions_repo.get_subscriptions_by_pattern(
Pattern('id'))
self.mocked_responses.add(
responses.GET,
'https://callback.url/1?hub.mode=unsubscribe&hub.topic=id&hub.challenge=UUID&hub.lease_seconds=432000',
body=self.MOCKED_UUID_VALUE)
params = {
'hub.mode': 'unsubscribe',
'hub.callback': 'https://callback.url/1',
'hub.topic': 'id',
}
response = self.do_subscribe_by_id_request(params)
assert response.status_code == 202, response.json
assert not self.subscriptions_repo.get_subscriptions_by_pattern(
Pattern('id'))
def test(notifications_repo, delivery_outbox_repo, subscriptions_repo):
config = Config()
processor = CallbackSpreader(config)
subscribers = ['subscriber/1', 'subscriber/2']
topic = 'aaa.bbb'
job = {'topic': topic, 'content': {'id': 'transaction-hash'}}
# creating test subscriptions
assert subscriptions_repo.is_empty()
for url in subscribers:
subscriptions_repo.subscribe_by_pattern(Pattern(topic), url, 300000)
# empty notifications_repo, processor must do nothing
assert notifications_repo.is_empty()
assert delivery_outbox_repo.is_empty()
next(processor)
assert notifications_repo.is_empty()
assert delivery_outbox_repo.is_empty()
# processor must pick the job from notifications repo
# with matching topic and create delivery job for each subscription
notifications_repo.post_job(job)
next(processor)
# checking delivery jobs
for i in range(2):
queue_job = delivery_outbox_repo.get_job()
assert queue_job
queue_msg_id, queue_job = queue_job
assert queue_job['s'] in subscribers
assert queue_job['payload'] == job['content']
assert queue_job['topic'] == topic
# checking that all delivery jobs tested
assert not delivery_outbox_repo.get_job()
# checking that notifications repo job was deleted after succesfully processing
assert notifications_repo.is_empty()
# this topic has no subscriptions, processor must not create delivery jobs
topic = 'aaa.ddd'
job = {'topic': topic, 'content': {'id': 'transaction-hash'}}
notifications_repo.post_job(job)
next(processor)
# checking that no delivery jobs were created
assert delivery_outbox_repo.is_empty()
# checking that notifications repo job was deleted after succesfully processing
assert notifications_repo.is_empty()
def test_subscribe_by_pattern__should_put_object_into_repo(self):
repo = SubscriptionsRepo(connection_data=self.connection_data)
pattern = Pattern('aaa.bbb.ccc')
repo.subscribe_by_pattern(pattern,
'http://callback.url/1',
expiration_seconds=3600)
self.client.put_object.assert_called_once()
args, kwargs = self.client.put_object.call_args
assert kwargs['Bucket'] == 'subscriptions'
assert kwargs['Key'] == 'AAA/BBB/CCC/ff0d1111f6636c354cf92c7137f1b5e6'
assert kwargs['Body'].read(
) == b'{"c": "http://callback.url/1", "e": "2020-05-18T15:08:00"}'
def test_get_subscriptions_by_pattern__when_same_callback__should_return_one(
self):
repo = SubscriptionsRepo(connection_data=self.connection_data)
self.client.list_objects.side_effect = [
{
'Contents': [{
'Key': 'AA/ff0d1111f6636c354cf92c7137f1b5e6'
}]
},
{
'Contents': [{
'Key': 'AA/BB/ff0d1111f6636c354cf92c7137f1b5e6'
}]
},
]
self.client.get_object.side_effect = [
{
'Body':
BytesIO(
b'{"c": "http://callback.url/1", "e": "2020-05-18T15:08:00"}'
),
'Bucket':
'subscriptions',
'ContentLength':
39,
'Key':
'AA',
},
{
'Body':
BytesIO(
b'{"c": "http://callback.url/1", "e": "2020-05-18T15:08:00"}'
),
'Bucket':
'subscriptions',
'ContentLength':
39,
'Key':
'AA/BB',
},
]
subscriptions = repo.get_subscriptions_by_pattern(Pattern('aa'))
assert len(subscriptions) == 1
def test_post__with_subscribe_mode__should_subscribe_to_all_messages_by_jurisdiction(
self, mocked_uuid):
self.mocked_responses.add(
responses.GET,
'https://callback.url/1?hub.mode=subscribe&hub.topic=jurisdiction.AU&hub.challenge=UUID&hub.lease_seconds=432000',
status=200,
body=self.MOCKED_UUID_VALUE)
params = {
'hub.mode': 'subscribe',
'hub.callback': 'https://callback.url/1',
'hub.topic': 'AU',
}
response = self.client.post(
url_for('views.subscriptions_by_jurisdiction'),
mimetype='application/x-www-form-urlencoded',
data=urlencode(params))
assert response.status_code == 202, response.json
assert self.subscriptions_repo.get_subscriptions_by_pattern(
Pattern('jurisdiction.AU'))
def test_to_key__when_predicate_valid__should_return_key(self):
assert Pattern('aaaa.bbbb.cccc').to_key() == "AAAA/BBBB/CCCC/"
def execute(self, topic):
try:
Pattern(topic)._validate()
return topic
except ValueError as e:
raise NotFoundError(detail='topic does not exist') from e
def execute(self, url, topic, expiration=None):
# this operation deletes all previous subscription for given url and pattern
# and replaces them with new one. Techically it's create or update operation
self.subscriptions_repo.subscribe_by_pattern(Pattern(topic), url,
expiration)
def test():
# testing predicate in url search
delivery_outbox_repo = repos.DeliveryOutboxRepo(DELIVERY_OUTBOX_REPO_CONF)
notifications_repo = repos.NotificationsRepo(NOTIFICATIONS_REPO_CONF)
subscriptions_repo = repos.SubscriptionsRepo(SUBSCRIPTIONS_REPO_CONF)
delivery_outbox_repo._unsafe_method__clear()
notifications_repo._unsafe_method__clear()
subscriptions_repo._unsafe_method__clear()
assert notifications_repo.is_empty()
assert delivery_outbox_repo.is_empty()
assert subscriptions_repo.is_empty()
use_case = DispatchMessageToSubscribersUseCase(notifications_repo, delivery_outbox_repo, subscriptions_repo)
processor = Processor(use_case=use_case)
# testing that iter returns processor
assert iter(processor) is processor
# assert processor has nothing to do
assert next(processor) is None
_fill_subscriptions_repo(subscriptions_repo, SUBSCRIPTIONS)
for prefix, subscriptions in SUBSCRIPTIONS_WITH_COMMON_PREFIXES.items():
_fill_subscriptions_repo(subscriptions_repo, subscriptions)
for s in subscriptions_repo.get_subscriptions_by_pattern(Pattern('aaa.bbb.ccc.ddd')):
print(s.__hash__())
print(s.payload, s.key, s.now, s.data)
# testing that subscriptions repod doesn't have side effect on processor
assert next(processor) is None
# testing callbacks spreading for predicates without common prefixes
for predicate, number_of_subscribers in SUBSCRIPTIONS.items():
# pushing notification
message = _generate_msg_object(predicate=predicate)
notifications_repo.post_job(message)
# test proccessor received notification
assert next(processor) is True
# test processor created correct number of delivery jobs
# each subscriptions group has unique topic/predicate
for i in range(number_of_subscribers):
job = delivery_outbox_repo.get_job()
assert job, f"Call:{i+1}. Predicate:{predicate}"
message_queue_id, payload = job
# test that only direct subscribers received this message
assert payload.get('payload', {}).get('predicate') == predicate
# testing that only correct subscribers will receive notification
url = payload.get('s', '')
assert _is_predicate_in_url(url, predicate), {'url': url, 'predicate': predicate}
assert delivery_outbox_repo.delete(message_queue_id)
# test queue is empty
print(delivery_outbox_repo.get_job())
assert not delivery_outbox_repo.get_job()
# processor completed the job
assert next(processor) is None
for prefix, subscriptions in SUBSCRIPTIONS_WITH_COMMON_PREFIXES.items():
# finding longest predicate in the group + total number of expected
# delivery jobs
expect_jobs = 0
longest_predicate = ""
for predicate, number_of_subscribers in subscriptions.items():
if len(longest_predicate) < len(predicate):
longest_predicate = predicate
expect_jobs += number_of_subscribers
# posting notification
message = _generate_msg_object(predicate=longest_predicate)
assert notifications_repo.post_job(message)
assert next(processor) is True
# testing processor created the correct number of delivery jobs
for i in range(expect_jobs):
job = delivery_outbox_repo.get_job()
assert job
message_queue_id, payload = job
# test that only direct subscribers received this message
assert payload.get('payload', {}).get('predicate') == longest_predicate
# testing that only correct subscribers will receive notification
url = payload.get('s', '')
assert _is_predicate_in_url(url, prefix), {'url': url, 'prefix': prefix}
assert delivery_outbox_repo.delete(message_queue_id)
assert next(processor) is None
def test_to_key__when_empty__should_return_error(self):
with pytest.raises(ValueError):
Pattern(topic='').to_key()
def test_to_layers__should_return_list_of_layers(self):
assert Pattern('aaaa.bbbb.cccc.*').to_layers() == [
'AAAA/', 'AAAA/BBBB/', 'AAAA/BBBB/CCCC/'
]
def test_to_key__with_url__should_add_url_hashed_as_suffix(self):
expected = "AAAA/BBBB/CCCC/8710bd9f92a413cbcaa13aa0e00953ba"
assert Pattern('aaaa.bbbb.cccc.*').to_key(
url='http://callback.com/1') == expected
def test_to_key__when_contains_slashes__should_return_error(self):
with pytest.raises(ValueError):
Pattern(topic='aa/bb').to_key()
def execute(self, callback=None, topic=None, expiration=None):
self.subscriptions_repo.subscribe_by_pattern(Pattern(topic), callback,
expiration)
def _get_subscriptions(self, topic):
subscribers = self.subscriptions.get_subscriptions_by_pattern(
Pattern(topic))
if not subscribers:
logger.info("Nobody to notify about the topic %s", topic)
return subscribers
def test_to_key__when_wildcard_without_dot__should_return_error(self):
with pytest.raises(ValueError):
Pattern(topic='aa.bb*').to_key()
def test_to_key__with_wildcard_in_predicate__should_be_handled(self):
assert Pattern('aaaa.bbbb.cccc.*').to_key() == "AAAA/BBBB/CCCC/"
assert Pattern('aaaa.bbbb.cccc.').to_key() == "AAAA/BBBB/CCCC/"