def test_ok_wifi_frequency(self):
app = self.app
wifi_data = [
{"key": "99"},
{"key": "aa", "frequency": 2427},
{"key": "bb", "channel": 7},
{"key": "cc", "frequency": 5200},
{"key": "dd", "frequency": 5700},
{"key": "ee", "frequency": 3100},
{"key": "ff", "frequency": 2412, "channel": 9},
]
res = app.post_json(
'/v1/submit', {"items": [{"lat": 12.345678,
"lon": 23.456789,
"wifi": wifi_data}]},
status=204)
self.assertEqual(res.body, '')
session = self.db_master_session
result = session.query(Measure).all()
self.assertEqual(len(result), 1)
item = result[0]
measure_wifi = loads(item.wifi)
measure_wifi = dict([(m['key'], m) for m in measure_wifi])
for k, v in measure_wifi.items():
self.assertFalse('frequency' in v)
self.assertEqual(measure_wifi['99']['channel'], 0)
self.assertEqual(measure_wifi['aa']['channel'], 4)
self.assertEqual(measure_wifi['bb']['channel'], 7)
self.assertEqual(measure_wifi['cc']['channel'], 40)
self.assertEqual(measure_wifi['dd']['channel'], 140)
self.assertEqual(measure_wifi['ee']['channel'], 0)
self.assertEqual(measure_wifi['ff']['channel'], 9)
def reprocess_cell_measure(self, measure_ids, userid=None):
measures = []
try:
with self.db_session() as session:
measures = session.query(Measure).filter(
Measure.id.in_(measure_ids)).filter(
Measure.cell != sql_null).all()
for measure in measures:
measure_data = dict(
id=measure.id, created=encode_datetime(measure.created),
lat=measure.lat, lon=measure.lon,
time=encode_datetime(measure.time),
accuracy=measure.accuracy, altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
radio=measure.radio,
)
# adds data to this session
process_cell_measure(
session, measure_data, loads(measure.cell), userid=userid)
session.commit()
return len(measures)
except IntegrityError as exc: # pragma: no cover
logger.exception('error')
return 0
except Exception as exc: # pragma: no cover
raise self.retry(exc=exc)
def preprocess_request(request, schema, extra_checks=(), response=JSONError):
body = {}
errors = []
validated = {}
body = request.body
if body:
if request.headers.get('Content-Encoding') == 'gzip':
# handle gzip request bodies
try:
body = zlib.decompress(body, 16 + zlib.MAX_WBITS)
except zlib.error:
errors.append(dict(name=None, description=MSG_GZIP))
if not errors:
try:
body = loads(body, encoding=request.charset)
except ValueError as e:
errors.append(dict(name=None, description=e.message))
else:
errors.append(dict(name=None, description=MSG_EMPTY))
if errors and response is not None:
# the response / None check is used in schema tests
# if we couldn't decode the JSON body, just return
raise response(errors)
# schema validation, but report at most one error at a time
schema = schema.bind(request=body)
for attr in schema.children:
name = attr.name
try:
if name not in body:
deserialized = attr.deserialize()
else:
deserialized = attr.deserialize(body[name])
except Invalid as e:
# the struct is invalid
err_dict = e.asdict()
try:
errors.append(dict(name=name, description=err_dict[name]))
break
except KeyError:
for k, v in err_dict.items():
if k.startswith(name):
errors.append(dict(name=k, description=v))
break
break
else:
validated[name] = deserialized
for func in extra_checks:
func(validated, errors)
if errors and response is not None:
# the response / None check is used in schema tests
request.registry.heka_client.debug('error_handler' + repr(errors))
raise response(errors)
return (validated, errors)
def test_ok_cell(self):
app = self.app
cell_data = [{
"radio": "umts",
"mcc": 123,
"mnc": 1,
"lac": 2,
"cid": 1234
}]
res = app.post_json('/v1/submit', {
"items": [{
"lat": 12.3456781,
"lon": 23.4567892,
"accuracy": 10,
"altitude": 123,
"altitude_accuracy": 7,
"radio": "gsm",
"cell": cell_data
}]
},
status=204)
self.assertEqual(res.body, '')
session = self.db_master_session
measure_result = session.query(Measure).all()
self.assertEqual(len(measure_result), 1)
item = measure_result[0]
self.assertEqual(item.lat, 123456781)
self.assertEqual(item.lon, 234567892)
self.assertEqual(item.accuracy, 10)
self.assertEqual(item.altitude, 123)
self.assertEqual(item.altitude_accuracy, 7)
self.assertEqual(item.radio, RADIO_TYPE['gsm'])
# colander schema adds default value
cell_data[0]['psc'] = 0
cell_data[0]['asu'] = 0
cell_data[0]['signal'] = 0
cell_data[0]['ta'] = 0
wanted = loads(item.cell)
self.assertTrue(len(wanted), 1)
self.assertTrue(len(cell_data), 1)
self.assertDictEqual(wanted[0], cell_data[0])
self.assertTrue(item.wifi is None)
cell_result = session.query(CellMeasure).all()
self.assertEqual(len(cell_result), 1)
item = cell_result[0]
self.assertEqual(item.measure_id, measure_result[0].id)
self.assertEqual(item.created, measure_result[0].created)
self.assertEqual(item.lat, 123456781)
self.assertEqual(item.lon, 234567892)
self.assertEqual(item.accuracy, 10)
self.assertEqual(item.altitude, 123)
self.assertEqual(item.altitude_accuracy, 7)
self.assertEqual(item.radio, RADIO_TYPE['umts'])
self.assertEqual(item.mcc, 123)
self.assertEqual(item.mnc, 1)
self.assertEqual(item.lac, 2)
self.assertEqual(item.cid, 1234)
def test_ok_wifi_frequency(self):
app = self.app
wifi_data = [
{
"key": "99"
},
{
"key": "aa",
"frequency": 2427
},
{
"key": "bb",
"channel": 7
},
{
"key": "cc",
"frequency": 5200
},
{
"key": "dd",
"frequency": 5700
},
{
"key": "ee",
"frequency": 3100
},
{
"key": "ff",
"frequency": 2412,
"channel": 9
},
]
res = app.post_json('/v1/submit', {
"items": [{
"lat": 12.345678,
"lon": 23.456789,
"wifi": wifi_data
}]
},
status=204)
self.assertEqual(res.body, '')
session = self.db_master_session
result = session.query(Measure).all()
self.assertEqual(len(result), 1)
item = result[0]
measure_wifi = loads(item.wifi)
measure_wifi = dict([(m['key'], m) for m in measure_wifi])
for k, v in measure_wifi.items():
self.assertFalse('frequency' in v)
self.assertEqual(measure_wifi['99']['channel'], 0)
self.assertEqual(measure_wifi['aa']['channel'], 4)
self.assertEqual(measure_wifi['bb']['channel'], 7)
self.assertEqual(measure_wifi['cc']['channel'], 40)
self.assertEqual(measure_wifi['dd']['channel'], 140)
self.assertEqual(measure_wifi['ee']['channel'], 0)
self.assertEqual(measure_wifi['ff']['channel'], 9)
def test_ok_cell(self):
app = self.app
cell_data = [
{"radio": "umts", "mcc": 123, "mnc": 1, "lac": 2, "cid": 1234}]
res = app.post_json(
'/v1/submit', {"items": [{"lat": 12.3456781,
"lon": 23.4567892,
"accuracy": 10,
"altitude": 123,
"altitude_accuracy": 7,
"radio": "gsm",
"cell": cell_data}]},
status=204)
self.assertEqual(res.body, '')
session = self.db_master_session
measure_result = session.query(Measure).all()
self.assertEqual(len(measure_result), 1)
item = measure_result[0]
self.assertEqual(item.lat, 123456781)
self.assertEqual(item.lon, 234567892)
self.assertEqual(item.accuracy, 10)
self.assertEqual(item.altitude, 123)
self.assertEqual(item.altitude_accuracy, 7)
self.assertEqual(item.radio, RADIO_TYPE['gsm'])
# colander schema adds default value
cell_data[0]['psc'] = 0
cell_data[0]['asu'] = 0
cell_data[0]['signal'] = 0
cell_data[0]['ta'] = 0
wanted = loads(item.cell)
self.assertTrue(len(wanted), 1)
self.assertTrue(len(cell_data), 1)
self.assertDictEqual(wanted[0], cell_data[0])
self.assertTrue(item.wifi is None)
cell_result = session.query(CellMeasure).all()
self.assertEqual(len(cell_result), 1)
item = cell_result[0]
self.assertEqual(item.measure_id, measure_result[0].id)
self.assertEqual(item.created, measure_result[0].created)
self.assertEqual(item.lat, 123456781)
self.assertEqual(item.lon, 234567892)
self.assertEqual(item.accuracy, 10)
self.assertEqual(item.altitude, 123)
self.assertEqual(item.altitude_accuracy, 7)
self.assertEqual(item.radio, RADIO_TYPE['umts'])
self.assertEqual(item.mcc, 123)
self.assertEqual(item.mnc, 1)
self.assertEqual(item.lac, 2)
self.assertEqual(item.cid, 1234)
def test_no_api_key(self):
app = self.app
session = self.db_slave_session
wifis = [
Wifi(key="A1", lat=10000000, lon=10000000, total_measures=9),
Wifi(key="B2", lat=10010000, lon=10020000, total_measures=9),
Wifi(key="C3", lat=10020000, lon=10040000, total_measures=9),
]
session.add_all(wifis)
session.commit()
res = app.post_json('/v1/search',
{"wifi": [
{"key": "A1"}, {"key": "B2"},
{"key": "C3"}, {"key": "D4"},
]},
status=400)
self.assertEqual(res.json, loads(NO_API_KEY))
self.check_expected_heka_messages(counter=['search.no_api_key'])
def insert_measures(self, items=None, nickname=''):
if not items: # pragma: no cover
return 0
items = loads(items)
length = len(items)
try:
with self.db_session() as session:
userid, nickname = process_user(nickname, session)
process_measures(items, session, userid=userid)
self.heka_client.incr("items.uploaded.batches", count=length)
session.commit()
return length
except IntegrityError as exc: # pragma: no cover
self.heka_client.raven('error')
return 0
except Exception as exc: # pragma: no cover
raise self.retry(exc=exc)
def insert_measures(self, items=None, nickname=''):
if not items: # pragma: no cover
return 0
items = loads(items)
length = len(items)
try:
with self.db_session() as session:
userid, nickname = process_user(nickname, session)
process_measures(items, session, userid=userid)
self.heka_client.incr("items.uploaded.batches", count=length)
session.commit()
return length
except IntegrityError as exc: # pragma: no cover
self.heka_client.raven('error')
return 0
except Exception as exc: # pragma: no cover
raise self.retry(exc=exc)