def fetch(items, queue, statsd):
messages = []
q_messages = queue.get_messages(num_messages=items)
statsd.incr('fetch.sqs.get_messages')
if len(q_messages) == 0:
statsd.incr('fetch.empty')
else:
statsd.incr('fetch.not_empty')
for q_m in q_messages:
try:
m = from_JSON(q_m.get_body())
except Exception, e:
# invalid JSON or values
if str(e) == 'Invalid JSON.':
statsd.incr('fetch.invalid.json')
continue
elif str(e) == 'Invalid data.':
statsd.incr('fetch.invalid.values')
continue
else:
raise e
messages.append(m)
statsd.incr('fetch.items')
def test_from_JSON(self):
tests = [
{# Invalid JSON
"message" : "{'metric' : 'users.registered', 'aggregation_type' : 'sum', 'start_time' : '01-04-2014 14:35:00', 'resolution' : '20sec', 'datapoints' : [1]}",
"expected" : Exception('Invalid JSON.')
},
{# Missing values
"message" : '{"metric" : "users.registered", "aggregation_type" : "sum", "start_time" : "01-04-2014 14:35:00", "resolution" : "20sec", "datapoints" : null}',
"expected" : Exception('Invalid data.')
},
{# Valid
"message" : '{"metric" : "users.registered", "aggregation_type" : "sum", "start_time" : "01-04-2014 14:35:00", "resolution" : "20sec", "datapoints" : [1.0, 2.0, 3.0]}',
"expected" : Message(metric = u"users.registered", aggregation_type = u"sum", start_time = u"01-04-2014 14:35:00", resolution = u"20sec", datapoints = [1.0, 2.0, 3.0])
}
]
test_counter = 0
for test in tests:
test_counter += 1
try:
out = from_JSON(test["message"])
except Exception, e:
out = e.args
test["expected"] = test["expected"].args
self.assertEquals(out, test["expected"], "[%d] Test expected %s, got %s" % (test_counter, test["expected"], out))
