def setUp(self):
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.router = SimpleRouter(settings, {})
self.notif = Notification(10, "data", dummy_chid, None, 200)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = settings.router = Mock(spec=Router)
self.storage_mock = settings.storage = Mock(spec=Storage)
self.agent_mock = Mock(spec=settings.agent)
settings.agent = self.agent_mock
self.router.metrics = Mock()
def setUp(self):
from twisted.logger import Logger
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.metrics = metrics = Mock(spec=SinkMetrics)
db = test_db(metrics=metrics)
self.agent_mock = agent = Mock(spec=Agent)
self.router = SimpleRouter(settings, {}, db, agent)
self.router.log = Mock(spec=Logger)
self.notif = Notification(10, "data", dummy_chid)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = db.router
self.storage_mock = db.storage
def setUp(self):
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.router = SimpleRouter(settings, {})
self.notif = Notification(10, "data", dummy_chid, None, 200)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = settings.router = Mock(spec=Router)
self.storage_mock = settings.storage = Mock(spec=Storage)
self.agent_mock = Mock(spec=settings.agent)
settings.agent = self.agent_mock
self.router.metrics = Mock()
class SimplePushRouterTestCase(unittest.TestCase):
def setUp(self):
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.router = SimpleRouter(settings, {})
self.notif = Notification(10, "data", dummy_chid, None, 200)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = settings.router = Mock(spec=Router)
self.storage_mock = settings.storage = Mock(spec=Storage)
self.agent_mock = Mock(spec=settings.agent)
settings.agent = self.agent_mock
self.router.metrics = Mock()
def tearDown(self):
dead_cache.clear()
def _raise_connect_error(self):
raise ConnectError()
def _raise_db_error(self):
raise ProvisionedThroughputExceededException(None, None)
def _raise_item_error(self):
raise ItemNotFound()
def test_register(self):
r = self.router.register(None, {})
eq_(r, {})
def test_route_to_connected(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 200
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 200)
d.addBoth(verify_deliver)
return d
def test_route_connect_error(self):
self.agent_mock.request.side_effect = MockAssist(
[self._raise_connect_error])
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.clear_node.return_value = None
d = self.router.route_notification(self.notif, router_data)
def verify_retry(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
eq_(len(self.router_mock.clear_node.mock_calls), 1)
self.router_mock.clear_node.reset_mock()
self.flushLoggedErrors()
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
eq_(len(self.router_mock.clear_node.mock_calls), 1)
self.router_mock.clear_node.reset_mock()
d = self.router.route_notification(self.notif, router_data)
d.addBoth(verify_retry)
return d
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_old_version(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = False
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_throws_db_error(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.side_effect = MockAssist(
[self._raise_db_error]
)
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_db_error]
)
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails_with_item_error(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_item_error]
)
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 404)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_no_node(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.return_value = dict()
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_202(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
assert(self.router_mock.get_uaid.called)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_send_check_fails(self):
import autopush.router.simple as simple
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
assert(self.router_mock.clear_node.called)
nk = simple.node_key(router_data["node_id"])
eq_(simple.dead_cache.get(nk), True)
d.addBoth(verify_deliver)
return d
def test_route_busy_node_saves_looks_up_and_send_check_fails_and_db(self):
import autopush.router.simple as simple
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
self.router_mock.clear_node.side_effect = MockAssist(
[self._raise_db_error]
)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
assert(self.router_mock.clear_node.called)
nk = simple.node_key(router_data["node_id"])
eq_(simple.dead_cache.get(nk), True)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_200(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.addCallback.return_value = response_mock
type(response_mock).code = PropertyMock(
side_effect=MockAssist([202, 200]))
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 200)
self.router.metrics.increment.assert_called_with(
"router.broadcast.save_hit"
)
d.addBoth(verify_deliver)
return d
@patch("requests.post")
def test_route_udp(self, request_mock):
self.storage_mock.save_notification.return_value = True
udp_data = {'wakeup_host': {'ip': '127.0.0.1', 'port': 9999},
'mobilenetwork': {'mcc': 'hammer'}}
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid,
udp=udp_data)
self.router_mock.get_uaid.return_value = router_data
self.router.conf = {'server': 'http://example.com',
'idle': 1, 'cert': 'test.pem'}
d = self.router.route_notification(self.notif, router_data)
def check_deliver(result):
eq_(result.status_code, 202)
d.addBoth(check_deliver)
eq_(self.router.udp, udp_data)
return d
def test_ammend(self):
resp = {"key": "value"}
eq_(resp, self.router.amend_msg(resp))
class SimplePushRouterTestCase(unittest.TestCase):
def setUp(self):
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.router = SimpleRouter(settings, {})
self.notif = Notification(10, "data", dummy_chid, None, 200)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = settings.router = Mock(spec=Router)
self.storage_mock = settings.storage = Mock(spec=Storage)
self.agent_mock = Mock(spec=settings.agent)
settings.agent = self.agent_mock
self.router.metrics = Mock()
def tearDown(self):
dead_cache.clear()
def _raise_connect_error(self):
raise ConnectError()
def _raise_db_error(self):
raise ProvisionedThroughputExceededException(None, None)
def _raise_item_error(self):
raise ItemNotFound()
def test_register(self):
r = self.router.register(None, {})
eq_(r, {})
def test_route_to_connected(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 200
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 200)
d.addBoth(verify_deliver)
return d
def test_route_connect_error(self):
self.agent_mock.request.side_effect = MockAssist(
[self._raise_connect_error])
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.clear_node.return_value = None
d = self.router.route_notification(self.notif, router_data)
def verify_retry(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
eq_(len(self.router_mock.clear_node.mock_calls), 1)
self.router_mock.clear_node.reset_mock()
self.flushLoggedErrors()
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
eq_(len(self.router_mock.clear_node.mock_calls), 1)
self.router_mock.clear_node.reset_mock()
d = self.router.route_notification(self.notif, router_data)
d.addBoth(verify_retry)
return d
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_old_version(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = False
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_throws_db_error(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.side_effect = MockAssist(
[self._raise_db_error]
)
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_db_error]
)
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails_with_item_error(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_item_error]
)
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 404)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_no_node(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.return_value = dict()
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_202(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
assert(self.router_mock.get_uaid.called)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_send_check_fails(self):
import autopush.router.simple as simple
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
assert(self.router_mock.clear_node.called)
nk = simple.node_key(router_data["node_id"])
eq_(simple.dead_cache.get(nk), True)
d.addBoth(verify_deliver)
return d
def test_route_busy_node_saves_looks_up_and_send_check_fails_and_db(self):
import autopush.router.simple as simple
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
self.router_mock.clear_node.side_effect = MockAssist(
[self._raise_db_error]
)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
assert(self.router_mock.clear_node.called)
nk = simple.node_key(router_data["node_id"])
eq_(simple.dead_cache.get(nk), True)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_200(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.addCallback.return_value = response_mock
type(response_mock).code = PropertyMock(
side_effect=MockAssist([202, 200]))
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 200)
self.router.metrics.increment.assert_called_with(
"router.broadcast.save_hit"
)
d.addBoth(verify_deliver)
return d
@patch("requests.post")
def test_route_udp(self, request_mock):
self.storage_mock.save_notification.return_value = True
udp_data = {'wakeup_host': {'ip': '127.0.0.1', 'port': 9999},
'mobilenetwork': {'mcc': 'hammer'}}
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid,
udp=udp_data)
self.router_mock.get_uaid.return_value = router_data
self.router.conf = {'server': 'http://example.com',
'idle': 1, 'cert': 'test.pem'}
d = self.router.route_notification(self.notif, router_data)
def check_deliver(result):
eq_(result.status_code, 202)
d.addBoth(check_deliver)
eq_(self.router.udp, udp_data)
return d
def test_ammend(self):
resp = {"key": "value"}
eq_(resp, self.router.amend_msg(resp))
def __init__(
self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf={},
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
s3_bucket=DEFAULT_BUCKET,
senderid_expry=SENDERID_EXPRY,
senderid_list={},
hello_timeout=0,
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
self.agent = Agent(reactor, connectTimeout=5, pool=pool)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
self.router_conf = router_conf
self.router_url = canonical_url(router_scheme or 'http',
self.router_hostname, router_port)
self.endpoint_url = canonical_url(endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port)
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_message_table(message_tablename,
message_read_throughput,
message_write_throughput)
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
self.message = Message(self.message_table, self.metrics)
# Run preflight check
preflight_check(self.storage, self.router)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = {}
self.routers["simplepush"] = SimpleRouter(
self, router_conf.get("simplepush"))
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
class SimplePushRouterTestCase(unittest.TestCase):
def setUp(self):
from twisted.logger import Logger
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.metrics = metrics = Mock(spec=SinkMetrics)
db = test_db(metrics=metrics)
self.agent_mock = agent = Mock(spec=Agent)
self.router = SimpleRouter(settings, {}, db, agent)
self.router.log = Mock(spec=Logger)
self.notif = Notification(10, "data", dummy_chid)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = db.router
self.storage_mock = db.storage
def _raise_connect_error(self):
raise ConnectError()
def _raise_connection_refused_error(self):
raise ConnectionRefusedError()
def _raise_db_error(self):
raise ProvisionedThroughputExceededException(None, None)
def _raise_item_error(self):
raise ItemNotFound()
def test_register(self):
router_data = {}
self.router.register("uaid", router_data=router_data, app_id="test123")
eq_(router_data, {})
def test_route_to_connected(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 200
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 200)
d.addBoth(verify_deliver)
return d
def test_route_connection_fail_saved(self):
self.agent_mock.request.side_effect = MockAssist(
[self._raise_connection_refused_error])
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.clear_node.return_value = None
self.router_mock.get_uaid.return_value = {}
self.storage_mock.save_notification.return_value = True
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(reply):
eq_(len(self.router.log.debug.mock_calls), 1)
ok_(reply.status_code, 202)
eq_(len(self.router_mock.clear_node.mock_calls), 1)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_old_version(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = False
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_throws_db_error(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.side_effect = MockAssist(
[self._raise_db_error])
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_db_error])
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails_with_item_error(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_item_error])
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 410)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_no_node(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.return_value = dict()
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_202(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
ok_(self.router_mock.get_uaid.called)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_send_check_fails(self):
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connection_refused_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
ok_(self.router_mock.clear_node.called)
d.addBoth(verify_deliver)
return d
def test_route_busy_node_saves_looks_up_and_send_check_fails_and_db(self):
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
self.router_mock.clear_node.side_effect = MockAssist(
[self._raise_db_error])
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 202)
ok_(self.router_mock.clear_node.called)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_200(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.addCallback.return_value = response_mock
type(response_mock).code = PropertyMock(
side_effect=MockAssist([202, 200]))
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(isinstance(result, RouterResponse))
eq_(result.status_code, 200)
d.addBoth(verify_deliver)
return d
@patch("requests.post")
def test_route_udp(self, request_mock):
self.storage_mock.save_notification.return_value = True
udp_data = {
'wakeup_host': {
'ip': '127.0.0.1',
'port': 9999
},
'mobilenetwork': {
'mcc': 'hammer'
}
}
router_data = dict(node_id="http://somewhere",
uaid=dummy_uaid,
udp=udp_data)
self.router_mock.get_uaid.return_value = router_data
self.router.conf = {
'server': 'http://example.com',
'idle': 1,
'cert': 'test.pem'
}
d = self.router.route_notification(self.notif, router_data)
def check_deliver(result):
eq_(result.status_code, 202)
d.addBoth(check_deliver)
eq_(self.router.udp, udp_data)
return d
def test_amend(self):
resp = {"key": "value"}
expected = resp.copy()
self.router.amend_endpoint_response(resp, {})
eq_(resp, expected)
def __init__(
self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf=None,
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
hello_timeout=0,
bear_hash_key=None,
preflight_uaid="deadbeef00000000deadbeef00000000",
ami_id=None,
client_certs=None,
msg_limit=100,
debug=False,
connect_timeout=0.5,
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
if not debug:
pool._factory = QuietClientFactory
self.agent = Agent(reactor, connectTimeout=connect_timeout, pool=pool)
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
if bear_hash_key is None:
bear_hash_key = []
if not isinstance(bear_hash_key, list):
bear_hash_key = [bear_hash_key]
self.bear_hash_key = bear_hash_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
hostname=self.hostname,
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval,
)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
if router_conf is None:
router_conf = {}
self.router_conf = router_conf
self.router_url = canonical_url(router_scheme or 'http',
self.router_hostname, router_port)
self.endpoint_url = canonical_url(endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port)
self.enable_tls_auth = client_certs is not None
self.client_certs = client_certs
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_rotating_message_table(
message_tablename,
message_read_throughput=message_read_throughput,
message_write_throughput=message_write_throughput)
self._message_prefix = message_tablename
self.message_limit = msg_limit
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
# Used to determine whether a connection is out of date with current
# db objects. There are three noteworty cases:
# 1 "Last Month" the table requires a rollover.
# 2 "This Month" the most common case.
# 3 "Next Month" where the system will soon be rolling over, but with
# timing, some nodes may roll over sooner. Ensuring the next month's
# table is present before the switchover is the main reason for this,
# just in case some nodes do switch sooner.
self.create_initial_message_tables()
# Run preflight check
preflight_check(self.storage, self.router, preflight_uaid)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = dict()
self.routers["simplepush"] = SimpleRouter(
self, router_conf.get("simplepush"))
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
self.ami_id = ami_id
# Generate messages per legacy rules, only used for testing to
# generate legacy data.
self._notification_legacy = False
class SimplePushRouterTestCase(unittest.TestCase):
def setUp(self):
settings = AutopushSettings(
hostname="localhost",
statsd_host=None,
)
self.router = SimpleRouter(settings, {})
self.notif = Notification(10, "data", dummy_chid)
mock_result = Mock(spec=gcmclient.gcm.Result)
mock_result.canonical = dict()
mock_result.failed = dict()
mock_result.not_registered = dict()
mock_result.needs_retry.return_value = False
self.router_mock = settings.router = Mock(spec=Router)
self.storage_mock = settings.storage = Mock(spec=Storage)
self.agent_mock = Mock(spec=settings.agent)
settings.agent = self.agent_mock
self.router.metrics = Mock()
def _raise_connect_error(self):
raise ConnectError()
def _raise_db_error(self):
raise ProvisionedThroughputExceededException(None, None)
def _raise_item_error(self):
raise ItemNotFound()
def test_register(self):
r = self.router.register(None, {})
eq_(r, {})
def test_route_to_connected(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 200
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 200)
d.addBoth(verify_deliver)
return d
def test_route_connect_error(self):
self.agent_mock.request.side_effect = MockAssist(
[self._raise_connect_error])
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.clear_node.return_value = None
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
self.flushLoggedErrors()
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_old_version(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = False
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_save_throws_db_error(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.side_effect = MockAssist(
[self._raise_db_error]
)
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 503)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_db_error]
)
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_with_no_node_saves_and_lookup_fails_with_item_error(self):
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.side_effect = MockAssist(
[self._raise_item_error]
)
router_data = dict(uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(fail):
exc = fail.value
ok_(exc, RouterException)
eq_(exc.status_code, 404)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_no_node(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
self.router_mock.get_uaid.return_value = dict()
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 202)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_202(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 202)
self.router_mock.get_uaid.assert_called()
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_send_check_fails(self):
import autopush.router.simple as simple
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 202)
self.router_mock.clear_node.assert_called()
nk = simple.node_key(router_data["node_id"])
eq_(simple.dead_cache.get(nk), True)
d.addBoth(verify_deliver)
return d
def test_route_busy_node_saves_looks_up_and_send_check_fails_and_db(self):
import autopush.router.simple as simple
response_mock = Mock()
self.agent_mock.request.side_effect = MockAssist(
[response_mock, self._raise_connect_error])
response_mock.code = 202
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
self.router_mock.clear_node.side_effect = MockAssist(
[self._raise_db_error]
)
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 202)
self.router_mock.clear_node.assert_called()
nk = simple.node_key(router_data["node_id"])
eq_(simple.dead_cache.get(nk), True)
d.addBoth(verify_deliver)
return d
def test_route_to_busy_node_saves_looks_up_and_sends_check_200(self):
self.agent_mock.request.return_value = response_mock = Mock()
response_mock.addCallback.return_value = response_mock
type(response_mock).code = PropertyMock(
side_effect=MockAssist([202, 200]))
self.storage_mock.save_notification.return_value = True
router_data = dict(node_id="http://somewhere", uaid=dummy_uaid)
self.router_mock.get_uaid.return_value = router_data
d = self.router.route_notification(self.notif, router_data)
def verify_deliver(result):
ok_(result, RouterResponse)
eq_(result.status_code, 200)
self.router.metrics.increment.assert_called_with(
"router.broadcast.save_hit"
)
d.addBoth(verify_deliver)
return d