def create_initial_message_tables(self):
"""Initializes a dict of the initial rotating messages tables.
An entry for last months table, and an entry for this months table.
"""
last_month = get_rotating_message_table(self._message_prefix, -1)
this_month = get_rotating_message_table(self._message_prefix)
self.message_tables = {
last_month.table_name: Message(last_month, self.metrics),
this_month.table_name: Message(this_month, self.metrics),
}
def test_message_storage(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
data1 = str(uuid.uuid4())
data2 = str(uuid.uuid4())
ttl = int(time.time()) + 100
time1, time2, time3 = self._nstime(), self._nstime(
), self._nstime() + 1
message.store_message(self.uaid, chid, time1, ttl, data1, {})
message.store_message(self.uaid, chid2, time2, ttl, data2, {})
message.store_message(self.uaid, chid2, time3, ttl, data1, {})
all_messages = list(message.fetch_messages(self.uaid))
eq_(len(all_messages), 3)
message.delete_messages_for_channel(self.uaid, chid2)
all_messages = list(message.fetch_messages(self.uaid))
eq_(len(all_messages), 1)
message.delete_message(self.uaid, chid, time1)
all_messages = list(message.fetch_messages(self.uaid))
eq_(len(all_messages), 0)
def test_unregister(self):
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Verify its in the db
rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
results = list(rows)
assert (len(results) == 1)
eq_(results[0]["chids"], set([chid]))
message.unregister_channel(self.uaid, chid)
# Verify its not in the db
rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
results = list(rows)
assert (len(results) == 1)
eq_(results[0]["chids"], set([]))
# Test for the very unlikely case that there's no 'chid'
m.connection.update_item = Mock()
m.connection.update_item.return_value = {
'Attributes': {
'uaid': {
'S': self.uaid
}
},
'ConsumedCapacityUnits': 0.5
}
r = message.unregister_channel(self.uaid, dummy_chid)
eq_(r, False)
def test_message_delete_pagination(self):
def make_messages(channel_id, count):
m = []
t = self._nstime()
ttl = int(time.time()) + 200
for i in range(count):
m.append(
(self.uaid, channel_id, str(uuid.uuid4()), ttl, {}, t + i))
return m
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Shove 80 messages in
for message_args in make_messages(chid, 80):
message.store_message(*message_args)
# Verify we can see them all
all_messages = list(message.fetch_messages(self.uaid, limit=100))
eq_(len(all_messages), 80)
# Delete them all
message.delete_messages_for_channel(self.uaid, chid)
# Verify they're gone
all_messages = list(message.fetch_messages(self.uaid, limit=100))
eq_(len(all_messages), 0)
def update_rotating_tables(self):
"""This method is intended to be tasked to run periodically off the
twisted event hub to rotate tables.
When today is a new month from yesterday, then we swap out all the
table objects on the settings object.
"""
today = datetime.date.today()
if today.month == self.current_month:
# No change in month, we're fine.
returnValue(False)
# Get tables for the new month, and verify they exist before we try to
# switch over
message_table = get_rotating_message_table(self._message_prefix)
try:
yield deferToThread(message_table.describe)
except Exception:
tblname = make_rotating_tablename(self._message_prefix)
log.err("Unable to locate new message table: %s" % tblname)
returnValue(False)
# Both tables found, safe to switch-over
self.current_month = today.month
self.current_msg_month = message_table.table_name
self.message_tables[self.current_msg_month] = \
Message(message_table, self.metrics)
returnValue(True)
def test_message_rotate_table_with_date(self):
prefix = "message" + uuid.uuid4().hex
future = datetime.today() + timedelta(days=32)
tbl_name = make_rotating_tablename(prefix, date=future)
m = get_rotating_message_table(prefix=prefix, date=future)
eq_(m.table_name, tbl_name)
def test_unregister(self):
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Verify its in the db
rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
results = list(rows)
assert len(results) == 1
eq_(results[0]["chids"], set([chid]))
message.unregister_channel(self.uaid, chid)
# Verify its not in the db
rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
results = list(rows)
assert len(results) == 1
eq_(results[0]["chids"], set([]))
# Test for the very unlikely case that there's no 'chid'
m.connection.update_item = Mock()
m.connection.update_item.return_value = {"Attributes": {"uaid": {"S": self.uaid}}, "ConsumedCapacityUnits": 0.5}
r = message.unregister_channel(self.uaid, "test")
eq_(r, False)
def test_message_storage(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
data1 = str(uuid.uuid4())
data2 = str(uuid.uuid4())
ttl = int(time.time()) + 100
time1, time2, time3 = self._nstime(), self._nstime(), self._nstime() + 1
message.store_message(self.uaid, chid, time1, ttl, data1, {})
message.store_message(self.uaid, chid2, time2, ttl, data2, {})
message.store_message(self.uaid, chid2, time3, ttl, data1, {})
all_messages = list(message.fetch_messages(self.uaid))
eq_(len(all_messages), 3)
message.delete_messages_for_channel(self.uaid, chid2)
all_messages = list(message.fetch_messages(self.uaid))
eq_(len(all_messages), 1)
message.delete_message(self.uaid, chid, time1)
all_messages = list(message.fetch_messages(self.uaid))
eq_(len(all_messages), 0)
def test_message_delete_pagination(self):
def make_messages(channel_id, count):
m = []
t = self._nstime()
ttl = int(time.time()) + 200
for i in range(count):
m.append((self.uaid, channel_id, str(uuid.uuid4()), ttl, {}, t + i))
return m
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Shove 80 messages in
for message_args in make_messages(chid, 80):
message.store_message(*message_args)
# Verify we can see them all
all_messages = list(message.fetch_messages(self.uaid, limit=100))
eq_(len(all_messages), 80)
# Delete them all
message.delete_messages_for_channel(self.uaid, chid)
# Verify they're gone
all_messages = list(message.fetch_messages(self.uaid, limit=100))
eq_(len(all_messages), 0)
def update_rotating_tables(self):
"""This method is intended to be tasked to run periodically off the
twisted event hub to rotate tables.
When today is a new month from yesterday, then we swap out all the
table objects on the settings object.
"""
today = datetime.date.today()
tomorrow = self._tomorrow()
if ((tomorrow.month != today.month)
and sorted(self.message_tables.keys())[-1] != tomorrow.month):
next_month = get_rotating_message_table(self._message_prefix, 0,
tomorrow)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
if today.month == self.current_month:
# No change in month, we're fine.
returnValue(False)
# Get tables for the new month, and verify they exist before we try to
# switch over
message_table = yield deferToThread(get_rotating_message_table,
self._message_prefix)
# Both tables found, safe to switch-over
self.current_month = today.month
self.current_msg_month = message_table.table_name
self.message_tables[self.current_msg_month] = \
Message(message_table, self.metrics)
returnValue(True)
def update_rotating_tables(self):
"""This method is intended to be tasked to run periodically off the
twisted event hub to rotate tables.
When today is a new month from yesterday, then we swap out all the
table objects on the settings object.
"""
today = datetime.date.today()
tomorrow = self._tomorrow()
if ((tomorrow.month != today.month) and
sorted(self.message_tables.keys())[-1] !=
tomorrow.month):
next_month = get_rotating_message_table(
self._message_prefix, 0, tomorrow)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
if today.month == self.current_month:
# No change in month, we're fine.
returnValue(False)
# Get tables for the new month, and verify they exist before we try to
# switch over
message_table = yield deferToThread(get_rotating_message_table,
self._message_prefix)
# Both tables found, safe to switch-over
self.current_month = today.month
self.current_msg_month = message_table.table_name
self.message_tables[self.current_msg_month] = \
Message(message_table, self.metrics)
returnValue(True)
def test_message_rotate_table_with_date(self):
prefix = "message" + uuid.uuid4().hex
future = (datetime.today() + timedelta(days=32)).date()
tbl_name = make_rotating_tablename(prefix, date=future)
m = get_rotating_message_table(prefix=prefix, date=future)
assert m.table_name == tbl_name
# Clean up the temp table.
_drop_table(tbl_name)
def test_register(self):
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Verify its in the db
rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
results = list(rows)
assert len(results) == 1
def create_initial_message_tables(self):
"""Initializes a dict of the initial rotating messages tables.
An entry for last months table, and an entry for this months table.
"""
last_month = get_rotating_message_table(self._message_prefix, -1)
this_month = get_rotating_message_table(self._message_prefix)
# Verify the table exists
try:
this_month.describe()
except Exception:
# Create the message table
create_rotating_message_table(prefix=self._message_prefix)
this_month = get_rotating_message_table(self._message_prefix)
self.message_tables = {
last_month.table_name: Message(last_month, self.metrics),
this_month.table_name: Message(this_month, self.metrics),
}
def test_register(self):
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Verify its in the db
rows = m.query_2(uaid__eq=self.uaid, chidmessageid__eq=" ")
results = list(rows)
assert (len(results) == 1)
def test_update_message_fail(self):
message = Message(get_rotating_message_table(), SinkMetrics)
message.store_message(self.uaid, uuid.uuid4().hex, self._nstime(), str(uuid.uuid4()), {})
u = message.table.connection.update_item = Mock()
def raise_condition(*args, **kwargs):
raise ConditionalCheckFailedException(None, None)
u.side_effect = raise_condition
b = message.update_message(self.uaid, uuid.uuid4().hex, self._nstime(), str(uuid.uuid4()), {})
eq_(b, False)
def test_preflight_check(self):
router = Router(get_router_table(), SinkMetrics())
message = Message(get_rotating_message_table(), SinkMetrics())
pf_uaid = "deadbeef00000000deadbeef01010101"
preflight_check(message, router, pf_uaid)
# now check that the database reports no entries.
_, notifs = message.fetch_messages(uuid.UUID(pf_uaid))
assert len(notifs) == 0
with pytest.raises(ItemNotFound):
router.get_uaid(pf_uaid)
def create_initial_message_tables(self):
"""Initializes a dict of the initial rotating messages tables.
An entry for last months table, an entry for this months table,
an entry for tomorrow, if tomorrow is a new month.
"""
today = datetime.date.today()
last_month = get_rotating_message_table(self._message_prefix, -1)
this_month = get_rotating_message_table(self._message_prefix)
self.current_month = today.month
self.current_msg_month = this_month.table_name
self.message_tables = {
last_month.table_name: Message(last_month, self.metrics),
this_month.table_name: Message(this_month, self.metrics)
}
if self._tomorrow().month != today.month:
next_month = get_rotating_message_table(delta=1)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
def test_message_delete_fail_condition(self):
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
def raise_condition(*args, **kwargs):
raise ConditionalCheckFailedException(None, None)
message.table = Mock()
message.table.delete_item.side_effect = raise_condition
result = message.delete_message(uaid="asdf", channel_id="asdf", message_id="asdf", updateid="asdf")
eq_(result, False)
def create_initial_message_tables(self):
"""Initializes a dict of the initial rotating messages tables.
An entry for last months table, an entry for this months table,
an entry for tomorrow, if tomorrow is a new month.
"""
today = datetime.date.today()
last_month = get_rotating_message_table(self._message_prefix, -1)
this_month = get_rotating_message_table(self._message_prefix)
self.current_month = today.month
self.current_msg_month = this_month.table_name
self.message_tables = {
last_month.table_name: Message(last_month, self.metrics),
this_month.table_name: Message(this_month, self.metrics)
}
if self._tomorrow().month != today.month:
next_month = get_rotating_message_table(delta=1)
self.message_tables[next_month.table_name] = Message(
next_month, self.metrics)
def test_unregister(self):
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Verify its in the db
response = m.query(
KeyConditions={
'uaid': {
'AttributeValueList': [self.uaid],
'ComparisonOperator': 'EQ'
},
'chidmessageid': {
'AttributeValueList': [" "],
'ComparisonOperator': 'EQ'
},
},
ConsistentRead=True,
)
results = list(response.get('Items'))
assert len(results) == 1
assert results[0]["chids"] == {chid}
message.unregister_channel(self.uaid, chid)
# Verify its not in the db
response = m.query(
KeyConditions={
'uaid': {
'AttributeValueList': [self.uaid],
'ComparisonOperator': 'EQ'
},
'chidmessageid': {
'AttributeValueList': [" "],
'ComparisonOperator': 'EQ'
},
},
ConsistentRead=True,
)
results = list(response.get('Items'))
assert len(results) == 1
assert results[0].get("chids") is None
# Test for the very unlikely case that there's no 'chid'
m.update_item = Mock(return_value={
'Attributes': {
'uaid': self.uaid
},
'ResponseMetaData': {}
})
r = message.unregister_channel(self.uaid, dummy_chid)
assert r is False
def test_preflight_check_fail(self):
router = Router(get_router_table(), SinkMetrics())
message = Message(get_rotating_message_table(), SinkMetrics())
def raise_exc(*args, **kwargs): # pragma: no cover
raise Exception("Oops")
router.clear_node = Mock()
router.clear_node.side_effect = raise_exc
with pytest.raises(Exception):
preflight_check(message, router)
def test_save_channels(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
exists, chans = message.all_channels(self.uaid)
new_uaid = uuid.uuid4().hex
message.save_channels(new_uaid, chans)
_, new_chans = message.all_channels(new_uaid)
eq_(chans, new_chans)
def test_save_channels(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
exists, chans = message.all_channels(self.uaid)
new_uaid = uuid.uuid4().hex
message.save_channels(new_uaid, chans)
_, new_chans = message.all_channels(new_uaid)
eq_(chans, new_chans)
def test_update_message(self):
chid = uuid.uuid4().hex
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
data1 = str(uuid.uuid4())
data2 = str(uuid.uuid4())
time1 = self._nstime()
time2 = self._nstime() + 100
ttl = self._nstime() + 1000
message.store_message(self.uaid, chid, time1, ttl, data1, {})
message.update_message(self.uaid, chid, time2, ttl, data2, {})
messages = list(message.fetch_messages(self.uaid))
eq_(data2, messages[0]["#dd"])
def test_message_delete_fail_condition(self):
notif = make_webpush_notification(dummy_uaid, dummy_chid)
notif.message_id = notif.update_id = dummy_uaid
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
def raise_condition(*args, **kwargs):
raise ConditionalCheckFailedException(None, None)
message.table = Mock()
message.table.delete_item.side_effect = raise_condition
result = message.delete_message(notif)
eq_(result, False)
def test_update_message(self):
chid = uuid.uuid4().hex
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
data1 = str(uuid.uuid4())
data2 = str(uuid.uuid4())
time1 = self._nstime()
time2 = self._nstime() + 100
ttl = self._nstime() + 1000
message.store_message(self.uaid, chid, time1, ttl, data1, {})
message.update_message(self.uaid, chid, time2, ttl, data2, {})
messages = list(message.fetch_messages(self.uaid))
eq_(data2, messages[0]['#dd'])
def test_message_delete_fail_condition(self):
notif = make_webpush_notification(dummy_uaid, dummy_chid)
notif.message_id = notif.update_id = dummy_uaid
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
def raise_condition(*args, **kwargs):
raise ClientError({}, 'delete_item')
message.table = Mock()
message.table.delete_item.side_effect = raise_condition
result = message.delete_message(notif)
assert result is False
def test_all_channels_fail(self):
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.table.get_item = Mock()
message.table.get_item.return_value = {
"ResponseMetadata": {
"HTTPStatusCode": 400
},
}
res = message.all_channels(self.uaid)
assert res == (False, set([]))
def test_preflight_check_wait(self):
router = Router(get_router_table(), SinkMetrics())
message = Message(get_rotating_message_table(), SinkMetrics())
values = ["PENDING", "ACTIVE"]
message.table_status = Mock(side_effect=values)
pf_uaid = "deadbeef00000000deadbeef01010101"
preflight_check(message, router, pf_uaid)
# now check that the database reports no entries.
_, notifs = message.fetch_messages(uuid.UUID(pf_uaid))
assert len(notifs) == 0
with pytest.raises(ItemNotFound):
router.get_uaid(pf_uaid)
def test_message_delete_fail_condition(self):
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
def raise_condition(*args, **kwargs):
raise ConditionalCheckFailedException(None, None)
message.table = Mock()
message.table.delete_item.side_effect = raise_condition
result = message.delete_message(uaid=dummy_uaid,
channel_id=dummy_chid,
message_id="asdf",
updateid="asdf")
eq_(result, False)
def test_update_message_fail(self):
message = Message(get_rotating_message_table(), SinkMetrics)
message.store_message(self.uaid,
uuid.uuid4().hex, self._nstime(),
str(uuid.uuid4()), {})
u = message.table.connection.update_item = Mock()
def raise_condition(*args, **kwargs):
raise ConditionalCheckFailedException(None, None)
u.side_effect = raise_condition
b = message.update_message(self.uaid,
uuid.uuid4().hex, self._nstime(),
str(uuid.uuid4()), {})
eq_(b, False)
def test_message_storage(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
message.store_message(make_webpush_notification(self.uaid, chid))
message.store_message(make_webpush_notification(self.uaid, chid))
message.store_message(make_webpush_notification(self.uaid, chid))
_, all_messages = message.fetch_timestamp_messages(
uuid.UUID(self.uaid), " ")
eq_(len(all_messages), 3)
def test_all_channels(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
_, chans = message.all_channels(self.uaid)
assert chid in chans
assert chid2 in chans
message.unregister_channel(self.uaid, chid2)
_, chans = message.all_channels(self.uaid)
assert chid2 not in chans
assert chid in chans
def test_all_channels(self):
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
_, chans = message.all_channels(self.uaid)
assert (chid in chans)
assert (chid2 in chans)
message.unregister_channel(self.uaid, chid2)
_, chans = message.all_channels(self.uaid)
assert (chid2 not in chans)
assert (chid in chans)
def test_update_rotating_tables_month_end(self):
"""Test that rotating adds next months table
This test is intended to ensure that if the next day is a new
month, then update_rotating_tables realizes this and add's
the new table to the message_tables.
A pre-requisite is that today cannot be the last day of
the current month. Therefore, we first sub in _tomorrow to
ensure it always appears as next month, and then remove
the new table create_initial_tables made so we can observe
update_rotating_tables add the new one.
Note that sorting message table keys to find the last month
does *not work* since the month digit is not zero-padded.
"""
today = datetime.date.today()
next_month = today.month + 1
next_year = today.year
if next_month > 12: # pragma: nocover
next_month = 1
next_year += 1
tomorrow = datetime.datetime(year=next_year, month=next_month, day=1)
conf = AutopushConfig(hostname="example.com", resolve_hostname=True)
db = DatabaseManager.from_config(conf)
db._tomorrow = Mock(return_value=tomorrow)
db.create_initial_message_tables()
# We should have 3 tables, one for next/this/last month
assert len(db.message_tables) == 3
# Grab next month's table name and remove it
next_month = get_rotating_message_table(conf.message_table.tablename,
delta=1)
db.message_tables.pop(next_month.table_name)
# Get the deferred back
d = db.update_rotating_tables()
def check_tables(result):
assert len(db.message_tables) == 3
assert next_month.table_name in db.message_tables
d.addCallback(check_tables)
return d
def test_message_storage_overwrite(self):
"""Test that store_message can overwrite existing messages which
can occur in some reconnect cases but shouldn't error"""
chid = str(uuid.uuid4())
chid2 = str(uuid.uuid4())
notif1 = make_webpush_notification(self.uaid, chid)
notif2 = make_webpush_notification(self.uaid, chid)
notif3 = make_webpush_notification(self.uaid, chid2)
notif2.message_id = notif1.message_id
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
message.register_channel(self.uaid, chid2)
message.store_message(notif1)
message.store_message(notif2)
message.store_message(notif3)
all_messages = list(message.fetch_messages(uuid.UUID(self.uaid)))
eq_(len(all_messages), 2)
def test_register(self):
chid = str(uuid.uuid4())
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
message.register_channel(self.uaid, chid)
# Verify it's in the db
response = m.query(
KeyConditions={
'uaid': {
'AttributeValueList': [self.uaid],
'ComparisonOperator': 'EQ'
},
'chidmessageid': {
'AttributeValueList': ['02'],
'ComparisonOperator': 'LT'
}
},
ConsistentRead=True,
)
assert len(response.get('Items'))
def test_all_channels_no_uaid(self):
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
exists, chans = message.all_channels("asdf")
assert chans == set([])
def __init__(self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf={},
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
s3_bucket=DEFAULT_BUCKET,
senderid_expry=SENDERID_EXPRY,
senderid_list={},
hello_timeout=0,
auth_key=None,
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
self.agent = Agent(reactor, connectTimeout=5, pool=pool)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval
)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
if auth_key is None:
auth_key = []
if not isinstance(auth_key, list):
auth_key = [auth_key]
self.auth_key = auth_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
self.router_conf = router_conf
self.router_url = canonical_url(
router_scheme or 'http',
self.router_hostname,
router_port
)
self.endpoint_url = canonical_url(
endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port
)
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(
storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_rotating_message_table(
message_tablename)
self._message_prefix = message_tablename
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
# Used to determine whether a connection is out of date with current
# db objects
self.current_msg_month = make_rotating_tablename(self._message_prefix)
self.current_month = datetime.date.today().month
self.create_initial_message_tables()
# Run preflight check
preflight_check(self.storage, self.router)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = {}
self.routers["simplepush"] = SimpleRouter(
self,
router_conf.get("simplepush")
)
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
def __init__(
self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf=None,
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
hello_timeout=0,
bear_hash_key=None,
preflight_uaid="deadbeef00000000deadbeef00000000",
ami_id=None,
client_certs=None,
msg_limit=100,
debug=False,
connect_timeout=0.5,
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
if not debug:
pool._factory = QuietClientFactory
self.agent = Agent(reactor, connectTimeout=connect_timeout, pool=pool)
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
if bear_hash_key is None:
bear_hash_key = []
if not isinstance(bear_hash_key, list):
bear_hash_key = [bear_hash_key]
self.bear_hash_key = bear_hash_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
hostname=self.hostname,
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval,
)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
if router_conf is None:
router_conf = {}
self.router_conf = router_conf
self.router_url = canonical_url(router_scheme or 'http',
self.router_hostname, router_port)
self.endpoint_url = canonical_url(endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port)
self.enable_tls_auth = client_certs is not None
self.client_certs = client_certs
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_rotating_message_table(
message_tablename,
message_read_throughput=message_read_throughput,
message_write_throughput=message_write_throughput)
self._message_prefix = message_tablename
self.message_limit = msg_limit
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
# Used to determine whether a connection is out of date with current
# db objects. There are three noteworty cases:
# 1 "Last Month" the table requires a rollover.
# 2 "This Month" the most common case.
# 3 "Next Month" where the system will soon be rolling over, but with
# timing, some nodes may roll over sooner. Ensuring the next month's
# table is present before the switchover is the main reason for this,
# just in case some nodes do switch sooner.
self.create_initial_message_tables()
# Run preflight check
preflight_check(self.storage, self.router, preflight_uaid)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = dict()
self.routers["simplepush"] = SimpleRouter(
self, router_conf.get("simplepush"))
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
self.ami_id = ami_id
# Generate messages per legacy rules, only used for testing to
# generate legacy data.
self._notification_legacy = False
def __init__(self,
crypto_key=None,
datadog_api_key=None,
datadog_app_key=None,
datadog_flush_interval=None,
hostname=None,
port=None,
router_scheme=None,
router_hostname=None,
router_port=None,
endpoint_scheme=None,
endpoint_hostname=None,
endpoint_port=None,
router_conf={},
router_tablename="router",
router_read_throughput=5,
router_write_throughput=5,
storage_tablename="storage",
storage_read_throughput=5,
storage_write_throughput=5,
message_tablename="message",
message_read_throughput=5,
message_write_throughput=5,
statsd_host="localhost",
statsd_port=8125,
resolve_hostname=False,
max_data=4096,
# Reflected up from UDP Router
wake_timeout=0,
env='development',
enable_cors=False,
s3_bucket=DEFAULT_BUCKET,
senderid_expry=SENDERID_EXPRY,
senderid_list={},
hello_timeout=0,
bear_hash_key=None,
preflight_uaid="deadbeef00000000deadbeef000000000",
):
"""Initialize the Settings object
Upon creation, the HTTP agent will initialize, all configured routers
will be setup and started, logging will be started, and the database
will have a preflight check done.
"""
# Use a persistent connection pool for HTTP requests.
pool = HTTPConnectionPool(reactor)
self.agent = Agent(reactor, connectTimeout=5, pool=pool)
# Metrics setup
if datadog_api_key:
self.metrics = DatadogMetrics(
api_key=datadog_api_key,
app_key=datadog_app_key,
flush_interval=datadog_flush_interval
)
elif statsd_host:
self.metrics = TwistedMetrics(statsd_host, statsd_port)
else:
self.metrics = SinkMetrics()
if not crypto_key:
crypto_key = [Fernet.generate_key()]
if not isinstance(crypto_key, list):
crypto_key = [crypto_key]
self.update(crypto_key=crypto_key)
self.crypto_key = crypto_key
if bear_hash_key is None:
bear_hash_key = []
if not isinstance(bear_hash_key, list):
bear_hash_key = [bear_hash_key]
self.bear_hash_key = bear_hash_key
self.max_data = max_data
self.clients = {}
# Setup hosts/ports/urls
default_hostname = socket.gethostname()
self.hostname = hostname or default_hostname
if resolve_hostname:
self.hostname = resolve_ip(self.hostname)
self.port = port
self.endpoint_hostname = endpoint_hostname or self.hostname
self.router_hostname = router_hostname or self.hostname
self.router_conf = router_conf
self.router_url = canonical_url(
router_scheme or 'http',
self.router_hostname,
router_port
)
self.endpoint_url = canonical_url(
endpoint_scheme or 'http',
self.endpoint_hostname,
endpoint_port
)
# Database objects
self.router_table = get_router_table(router_tablename,
router_read_throughput,
router_write_throughput)
self.storage_table = get_storage_table(
storage_tablename,
storage_read_throughput,
storage_write_throughput)
self.message_table = get_rotating_message_table(
message_tablename)
self._message_prefix = message_tablename
self.storage = Storage(self.storage_table, self.metrics)
self.router = Router(self.router_table, self.metrics)
# Used to determine whether a connection is out of date with current
# db objects. There are three noteworty cases:
# 1 "Last Month" the table requires a rollover.
# 2 "This Month" the most common case.
# 3 "Next Month" where the system will soon be rolling over, but with
# timing, some nodes may roll over sooner. Ensuring the next month's
# table is present before the switchover is the main reason for this,
# just in case some nodes do switch sooner.
self.create_initial_message_tables()
# Run preflight check
preflight_check(self.storage, self.router, preflight_uaid)
# CORS
self.cors = enable_cors
# Force timeout in idle seconds
self.wake_timeout = wake_timeout
# Setup the routers
self.routers = {}
self.routers["simplepush"] = SimpleRouter(
self,
router_conf.get("simplepush")
)
self.routers["webpush"] = WebPushRouter(self, None)
if 'apns' in router_conf:
self.routers["apns"] = APNSRouter(self, router_conf["apns"])
if 'gcm' in router_conf:
self.routers["gcm"] = GCMRouter(self, router_conf["gcm"])
# Env
self.env = env
self.hello_timeout = hello_timeout
def setUp(self):
table = get_rotating_message_table()
self.real_table = table
self.real_connection = table.connection
self.uaid = str(uuid.uuid4())
def setUp(self):
table = get_rotating_message_table()
self.real_table = table
self.real_connection = table.connection
self.uaid = str(uuid.uuid4())
def test_all_channels_no_uaid(self):
m = get_rotating_message_table()
message = Message(m, SinkMetrics())
exists, chans = message.all_channels(dummy_uaid)
assert (chans == set([]))