def test_customjson(self): now = util.utcnow() report_id = uuid.uuid1() obs = CellObservation.create( radio=Radio.gsm, mcc=GB_MCC, mnc=5, lac=12345, cid=23456, report_id=report_id, lat=GB_LAT, lon=GB_LON, created=now) json_data = kombu_dumps(obs) self.assertTrue('accuracy' not in json_data) result = kombu_loads(json_data) self.assertTrue(type(result), CellObservation) self.assertTrue(result.accuracy is None) self.assertEqual(type(result.report_id), uuid.UUID) self.assertEqual(result.report_id, report_id) self.assertEqual(type(result.radio), Radio) self.assertEqual(result.radio, Radio.gsm) self.assertEqual(result.mcc, GB_MCC) self.assertEqual(result.mnc, 5) self.assertEqual(result.lac, 12345) self.assertEqual(result.cid, 23456) self.assertEqual(result.lat, GB_LAT) self.assertEqual(result.lon, GB_LON) self.assertEqual(type(result.created), datetime.datetime) self.assertEqual(result.created, now)
def _enqueue(self, items, queue_key, batch=100, expire=False, pipe=None): data = [str(kombu_dumps(item)) for item in items] if pipe is not None: self._push(pipe, data, queue_key, batch=batch, expire=expire) else: with redis_pipeline(self.redis_client) as pipe: self._push(pipe, data, queue_key, batch=batch, expire=expire)
def geosubmit_view(request): stats_client = request.registry.stats_client api_key_log = getattr(request, 'api_key_log', False) api_key_name = getattr(request, 'api_key_name', None) try: data, errors = preprocess_request( request, schema=GeoSubmitBatchSchema(), response=JSONParseError, ) except JSONParseError: # capture JSON exceptions for submit calls request.registry.heka_client.raven(RAVEN_ERROR) raise items = map_items(data['items']) nickname = request.headers.get('X-Nickname', u'') if isinstance(nickname, str): nickname = nickname.decode('utf-8', 'ignore') email = request.headers.get('X-Email', u'') if isinstance(email, str): email = email.decode('utf-8', 'ignore') # count the number of batches and emit a pseudo-timer to capture # the number of reports per batch length = len(items) stats_client.incr('items.uploaded.batches') stats_client.timing('items.uploaded.batch_size', length) if api_key_log: stats_client.incr('items.api_log.%s.uploaded.batches' % api_key_name) stats_client.timing( 'items.api_log.%s.uploaded.batch_size' % api_key_name, length) # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request for i in range(0, length, 100): batch = kombu_dumps(items[i:i + 100]) # insert observations, expire the task if it wasn't processed # after six hours to avoid queue overload try: insert_measures.apply_async(kwargs={ 'email': email, 'items': batch, 'nickname': nickname, 'api_key_log': api_key_log, 'api_key_name': api_key_name, }, expires=21600) except ConnectionError: # pragma: no cover return HTTPServiceUnavailable() result = HTTPOk() result.content_type = 'application/json' result.body = '{}' return result
def enqueue_areas(session, redis_client, area_keys, pipeline_key, expire=86400, batch=100): pipe = redis_client.pipeline() area_json = [str(kombu_dumps(area)) for area in area_keys] while area_json: pipe.lpush(pipeline_key, *area_json[:batch]) area_json = area_json[batch:] # Expire key after 24 hours pipe.expire(pipeline_key, expire) pipe.execute()
def insert_measures(self, request_data): # batch incoming data into multiple tasks, in case someone # manages to submit us a huge single request submit_data = self.prepare_measure_data(request_data) for i in range(0, len(submit_data), 100): batch = kombu_dumps(submit_data[i:i + 100]) # insert observations, expire the task if it wasn't processed # after six hours to avoid queue overload insert_measures.apply_async(kwargs={ 'api_key_text': self.api_key.valid_key, 'email': self.email, 'items': batch, 'nickname': self.nickname, }, expires=21600)
def queue_export(self, reports): metadata = { 'api_key': self.api_key, 'email': self.email, 'nickname': self.nickname, } data = [] for report in reports: data.append(str(kombu_dumps({'report': report, 'metadata': metadata}))) if data: for name, settings in self.export_queues.items(): redis_key = settings['redis_key'] source_apikey = settings.get('source_apikey', _sentinel) if self.api_key != source_apikey: self.redis_client.lpush(redis_key, *data)
def enqueue_lacs(session, redis_client, lac_keys, pipeline_key, expire=86400, batch=100): pipe = redis_client.pipeline() lac_json = [str(kombu_dumps(lac)) for lac in lac_keys] while lac_json: pipe.lpush(pipeline_key, *lac_json[:batch]) lac_json = lac_json[batch:] # Expire key after 24 hours pipe.expire(pipeline_key, expire) pipe.execute()
def send(self, url, data): groups = defaultdict(list) for item in simplejson.loads(data): group = MetadataGroup(**item['metadata']) report = self._format_report(item['report']) if report: groups[group].append(report) for group, reports in groups.items(): self._task().apply_async( kwargs={ 'api_key_text': group.api_key, 'email': group.email, 'ip': group.ip, 'items': kombu_dumps(reports), 'nickname': group.nickname, }, expires=21600)
def queue_export(self, reports): metadata = { 'api_key': self.api_key, 'email': self.email, 'nickname': self.nickname, } data = [] for report in reports: data.append( str(kombu_dumps({ 'report': report, 'metadata': metadata }))) if data: for name, settings in self.export_queues.items(): redis_key = settings['redis_key'] source_apikey = settings.get('source_apikey', _sentinel) if self.api_key != source_apikey: self.redis_client.lpush(redis_key, *data)
def test_customjson(self): key = '3680873e9b83' now = util.utcnow() report_id = uuid.uuid1() obs = WifiObservation.create( key=key, report_id=report_id, lat=GB_LAT, lon=GB_LON, created=now) json_data = kombu_dumps(obs) self.assertTrue('accuracy' not in json_data) result = kombu_loads(json_data) self.assertTrue(type(result), WifiObservation) self.assertTrue(result.accuracy is None) self.assertEqual(type(result.report_id), uuid.UUID) self.assertEqual(result.report_id, report_id) self.assertEqual(result.key, key) self.assertEqual(result.lat, GB_LAT) self.assertEqual(result.lon, GB_LON) self.assertEqual(type(result.created), datetime.datetime) self.assertEqual(result.created, now)
def test_datetime_us_roundtrip(self): us = pytz.timezone('US/Eastern') test_date = datetime(2012, 5, 17, 14, 28, 56, tzinfo=us) data = kombu_loads(kombu_dumps({'d': test_date})) self.assertEqual(test_date, data['d']) self.assertTrue(data['d'].tzinfo is pytz.utc)
def test_datetime_utc_roundtrip(self): test_date = util.utcnow() data = kombu_loads(kombu_dumps({'d': test_date})) self.assertEqual(test_date, data['d'])
def test_datetime_roundtrip(self): test_date = datetime(2012, 5, 17, 14, 28, 56) data = kombu_loads(kombu_dumps({'d': test_date})) self.assertEqual(test_date.replace(tzinfo=pytz.UTC), data['d'])
def test_datetime_dump(self): data = kombu_dumps({'d': datetime(2012, 5, 17, 14, 28, 56)}) self.assertTrue('__datetime__' in data)
def test_date_roundtrip(self): test_date = date(2012, 5, 17) data = kombu_loads(kombu_dumps({'d': test_date})) self.assertEqual(test_date, data['d'])
def test_uuid1_roundtrip(self): test_uuid = uuid.uuid1() data = kombu_loads(kombu_dumps({'d': test_uuid})) self.assertEqual(data['d'], test_uuid) self.assertEqual(data['d'].version, 1)
def test_namedtuple(self): Named = namedtuple('Named', 'one two') data = kombu_loads(kombu_dumps({'d': Named(one=1, two=[2])})) self.assertEqual(data['d'], {'one': 1, 'two': [2]})
def test_uuid4(self): data = kombu_dumps({'d': uuid.uuid4()}) self.assertTrue('__uuid__' in data)
def test_uuid4_roundtrip(self): test_uuid = uuid.uuid4() data = kombu_loads(kombu_dumps({'d': test_uuid})) self.assertEqual(data['d'], test_uuid) self.assertEqual(data['d'].version, 4)
def test_date_dump(self): data = kombu_dumps({'d': date(2012, 5, 17)}) self.assertTrue('__date__' in data)