def main(): statsd = StatsClient(host='localhost', port=8125, prefix=None, maxudpsize=512) queue = get_SQSqueue() db_connection = get_dynamoDBconnection() while True: messages = fetch(items=MAX_ITEMS, queue=queue, statsd=statsd) for message in messages: if apply2(message=message, connection=db_connection, statsd=statsd): pass
def test_apply(self):
tests = [
{# fresh write, table doesn't exist
'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 3, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.create',
'apply.dynamodb.table.get_item',
'apply.metric.create',
'apply.dynamodb.item.write',
'apply.sqs.indexer.write'
],
'expected_indexer_message' : {'body' : 'users.registered', 'resolution' : '20sec'}
},
{# fresh write, table exists
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : []
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 3, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.metric.create',
'apply.dynamodb.item.write',
'apply.sqs.indexer.write'
]
},
{# sum write, 0s
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 0]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[2, 3, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.sum'
]
},
{# sum write, existing None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, null]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[2, 3, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.sum'
]
},
{# sum write, None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 3, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, None]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[2, 3, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.sum'
]
},
{# average write, existing None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'average', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 0, 0]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 3]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.average'
]
},
{# average write, None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'average', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, 0]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 3]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.average'
]
},
{# minimum write, existing None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'minimum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 0, 0]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 0]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.minimum'
]
},
{# minimum write, None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'minimum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, 0]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 0]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.minimum'
]
},
{# maximum write, existing None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'maximum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 0, 0]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.maximum'
]
},
{# maximum write, None
'existing_data' : {
'table_name' : self.twenty_sec_table_name,
'items' : [
{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}
]
},
'change' : TMessage(metric = 'users.registered', aggregation_type = 'maximum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, 0]),
'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}},
'expected_stats' : [
'apply.dynamodb.table.describe',
'apply.dynamodb.table.get_item',
'apply.dynamodb.item.write',
'apply.metric.update',
'apply.aggregate.maximum'
]
}
]
test_counter = 0
for test in tests:
test_counter += 1
if test.has_key('existing_data'):
populate_tables(self.dynamodb_connection, test['existing_data'])
self.statsd._stats = []
apply2(
message=test['change'],
dynamodb_connection=self.dynamodb_connection,
indexer_queue=self.indexer_queue,
statsd=self.statsd
)
e = test['expected']['item']
out = get_metric(test['expected']['table_name'], self.dynamodb_connection, e['metric'], e['start_time'])
self.assertDictEqual(out, test['expected']['item'], '[%d] Test expected %s, got %s' % (test_counter, test['expected']['item'], out))
assertStatsd(self, self.statsd, test['expected_stats'], test_counter, '[%d] Test expected %s, got %s')
if test.has_key('expected_indexer_message'):
indexer_messages = self.indexer_queue.get_messages(message_attributes=['*'])
self.assertEquals(len(indexer_messages), 1, '[%d] Test expected an indexer message, got none.' % test_counter)
indexer_message = {'body' : indexer_messages[0].get_body(), 'resolution' : indexer_messages[0].message_attributes['resolution']['string_value']}
self.assertDictEqual(indexer_message, test['expected_indexer_message'], '[%d] Test expected %s, got %s' % (test_counter, test['expected_indexer_message'], indexer_message))
self.indexer_queue.clear()
self.tear_down_table()