Exemple #1
0
def main():

	statsd = StatsClient(host='localhost', port=8125, prefix=None, maxudpsize=512)

	queue = get_SQSqueue()
	db_connection = get_dynamoDBconnection()

	while True:
		messages = fetch(items=MAX_ITEMS, queue=queue, statsd=statsd)

		for message in messages:
			if apply2(message=message, connection=db_connection, statsd=statsd):
				pass
Exemple #2
0
	def test_apply(self):

		tests = [
			{# fresh write, table doesn't exist
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 3, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.create',
					'apply.dynamodb.table.get_item',
					'apply.metric.create',
					'apply.dynamodb.item.write',
					'apply.sqs.indexer.write'
				],
				'expected_indexer_message' : {'body' : 'users.registered', 'resolution' : '20sec'}
			},
			{# fresh write, table exists
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : []
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 3, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.metric.create',
					'apply.dynamodb.item.write',
					'apply.sqs.indexer.write'
				]
			},
			{# sum write, 0s
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 0]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[2, 3, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.sum'
				]
			},
			{# sum write, existing None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, null]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 3, 6]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[2, 3, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.sum'
				]
			},
			{# sum write, None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 3, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'sum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, None]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[2, 3, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.sum'
				]
			},
			{# average write, existing None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'average', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 0, 0]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 3]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.average'
				]
			},
			{# average write, None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'average', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, 0]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 3]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.average'
				]
			},
			{# minimum write, existing None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'minimum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 0, 0]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 0]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.minimum'
				]
			},
			{# minimum write, None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'minimum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, 0]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 0]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.minimum'
				]
			},
			{# maximum write, existing None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, null, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'maximum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, 0, 0]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.maximum'
				]
			},
			{# maximum write, None
				'existing_data' : {
					'table_name' : self.twenty_sec_table_name,
					'items' : [
						{'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}
					]
				},
				'change' : TMessage(metric = 'users.registered', aggregation_type = 'maximum', start_time = '01-04-2014 14:35:00', resolution = '20sec', datapoints = [1, None, 0]),
				'expected' : {'table_name' : self.twenty_sec_table_name, 'item' : {'metric' : 'users.registered', 'start_time' : '01-04-2014 14:35:00', 'datapoints' : '[1, 0, 6]'}},
				'expected_stats' : [
					'apply.dynamodb.table.describe',
					'apply.dynamodb.table.get_item',
					'apply.dynamodb.item.write',
					'apply.metric.update',
					'apply.aggregate.maximum'
				]
			}
		]

		test_counter = 0
		for test in tests:
			test_counter += 1
			
			if test.has_key('existing_data'):
				populate_tables(self.dynamodb_connection, test['existing_data'])

			self.statsd._stats = []

			apply2(
				message=test['change'],
				dynamodb_connection=self.dynamodb_connection,
				indexer_queue=self.indexer_queue,
				statsd=self.statsd
			)
			
			e = test['expected']['item']

			out = get_metric(test['expected']['table_name'], self.dynamodb_connection, e['metric'], e['start_time'])

			self.assertDictEqual(out, test['expected']['item'], '[%d] Test expected %s, got %s' % (test_counter, test['expected']['item'], out))
			assertStatsd(self, self.statsd, test['expected_stats'], test_counter, '[%d] Test expected %s, got %s')
			
			if test.has_key('expected_indexer_message'):

				indexer_messages = self.indexer_queue.get_messages(message_attributes=['*'])
				self.assertEquals(len(indexer_messages), 1, '[%d] Test expected an indexer message, got none.' % test_counter)

				indexer_message = {'body' : indexer_messages[0].get_body(), 'resolution' : indexer_messages[0].message_attributes['resolution']['string_value']}
				self.assertDictEqual(indexer_message, test['expected_indexer_message'], '[%d] Test expected %s, got %s' % (test_counter, test['expected_indexer_message'], indexer_message))

			self.indexer_queue.clear()
			self.tear_down_table()