def process_measure(data, utcnow, session):
session_objects = []
measure = Measure()
measure.created = utcnow
measure.time = data['time']
measure.lat = to_precise_int(data['lat'])
measure.lon = to_precise_int(data['lon'])
measure.accuracy = data['accuracy']
measure.altitude = data['altitude']
measure.altitude_accuracy = data['altitude_accuracy']
measure.radio = RADIO_TYPE.get(data['radio'], -1)
# get measure.id set
session.add(measure)
session.flush()
if data.get('cell'):
cells, cell_data = process_cell(data['cell'], measure)
measure.cell = dumps(cell_data)
session_objects.extend(cells)
if data.get('wifi'):
# filter out old-style sha1 hashes
too_long_keys = False
for w in data['wifi']:
w['key'] = key = normalize_wifi_key(w['key'])
if len(key) > 12:
too_long_keys = True
break
if not too_long_keys:
process_wifi(data['wifi'], measure)
measure.wifi = dumps(data['wifi'])
return (measure, session_objects)
def submit_view(request):
api_key = request.GET.get('key', None)
heka_client = get_heka_client()
if api_key is None:
# we don't require API keys for submit yet
heka_client.incr('submit.no_api_key')
else:
heka_client.incr('submit.api_key.%s' % api_key.replace('.', '__'))
data, errors = preprocess_request(
request,
schema=SubmitSchema(),
extra_checks=(submit_validator, ),
)
items = data['items']
nickname = request.headers.get('X-Nickname', u'')
if isinstance(nickname, str):
nickname = nickname.decode('utf-8', 'ignore')
# batch incoming data into multiple tasks, in case someone
# manages to submit us a huge single request
for i in range(0, len(items), 100):
insert_measures.delay(
# TODO convert items to json with support for decimal/datetime
items=dumps(items[i:i + 100]),
nickname=nickname,
)
return HTTPNoContent()
def submit_view(request):
api_key = request.GET.get('key', None)
heka_client = get_heka_client()
if api_key is None:
# we don't require API keys for submit yet
heka_client.incr('submit.no_api_key')
else:
heka_client.incr('submit.api_key.%s' % api_key.replace('.', '__'))
data, errors = preprocess_request(
request,
schema=SubmitSchema(),
extra_checks=(submit_validator, ),
)
items = data['items']
nickname = request.headers.get('X-Nickname', u'')
if isinstance(nickname, str):
nickname = nickname.decode('utf-8', 'ignore')
# batch incoming data into multiple tasks, in case someone
# manages to submit us a huge single request
for i in range(0, len(items), 100):
insert_measures.delay(
# TODO convert items to json with support for decimal/datetime
items=dumps(items[i:i + 100]),
nickname=nickname,
)
return HTTPNoContent()
def test_gzip(self):
app = self.app
data = {"cell": [{"mcc": 1, "mnc": 2, "lac": 3, "cid": 4}]}
body = ''.join(gzip_app_iter(dumps(data)))
headers = {
'Content-Encoding': 'gzip',
}
res = app.post('/v1/search?key=test', body, headers=headers,
content_type='application/json', status=200)
self.assertEqual(res.content_type, 'application/json')
self.assertEqual(res.json, {"status": "not_found"})
def test_gzip(self):
app = self.app
data = {"items": [{"lat": 1.0,
"lon": 2.0,
"wifi": [{"key": "aaaaaaaaaaaa"}]}]}
body = ''.join(gzip_app_iter(dumps(data)))
headers = {
'Content-Encoding': 'gzip',
}
res = app.post('/v1/submit?key=test', body, headers=headers,
content_type='application/json', status=204)
self.assertEqual(res.body, '')
def process_measure(data, utcnow, session, userid=None):
measure = Measure()
measure.created = utcnow
measure.time = data["time"]
measure.lat = to_precise_int(data["lat"])
measure.lon = to_precise_int(data["lon"])
measure.accuracy = data["accuracy"]
measure.altitude = data["altitude"]
measure.altitude_accuracy = data["altitude_accuracy"]
measure.radio = RADIO_TYPE.get(data["radio"], -1)
# get measure.id set
session.add(measure)
session.flush()
measure_data = dict(
id=measure.id,
created=encode_datetime(measure.created),
lat=measure.lat,
lon=measure.lon,
time=encode_datetime(measure.time),
accuracy=measure.accuracy,
altitude=measure.altitude,
altitude_accuracy=measure.altitude_accuracy,
radio=measure.radio,
)
if data.get("cell"):
insert_cell_measure.delay(measure_data, data["cell"], userid=userid)
measure.cell = dumps(data["cell"])
if data.get("wifi"):
# filter out old-style sha1 hashes
too_long_keys = False
for w in data["wifi"]:
w["key"] = key = normalize_wifi_key(w["key"])
if len(key) > 12:
too_long_keys = True
break
if not too_long_keys:
insert_wifi_measure.delay(measure_data, data["wifi"], userid=userid)
measure.wifi = dumps(data["wifi"])
return measure
def submit_view(request):
data, errors = preprocess_request(
request,
schema=SubmitSchema(),
extra_checks=(submit_validator, ),
)
items = data['items']
nickname = request.headers.get('X-Nickname', u'')
if isinstance(nickname, str):
nickname = nickname.decode('utf-8', 'ignore')
# batch incoming data into multiple tasks, in case someone
# manages to submit us a huge single request
for i in range(0, len(items), 100):
insert_measures.delay(
items=dumps(items[i:i + 100]),
nickname=nickname,
)
return HTTPNoContent()
def submit_view(request):
data, errors = preprocess_request(
request,
schema=SubmitSchema(),
extra_checks=(submit_validator, ),
)
items = data['items']
nickname = request.headers.get('X-Nickname', u'')
if isinstance(nickname, str):
nickname = nickname.decode('utf-8', 'ignore')
# batch incoming data into multiple tasks, in case someone
# manages to submit us a huge single request
for i in range(0, len(items), 100):
insert_measures.delay(
# TODO convert items to json with support for decimal/datetime
items=dumps(items[i:i + 100]),
nickname=nickname,
)
return HTTPNoContent()
def test_gzip(self):
app = self.app
data = {
"items": [{
"lat": 1.0,
"lon": 2.0,
"wifi": [{
"key": "aaaaaaaaaaaa"
}]
}]
}
body = ''.join(gzip_app_iter(dumps(data)))
headers = {
'Content-Encoding': 'gzip',
}
res = app.post('/v1/submit?key=test',
body,
headers=headers,
content_type='application/json',
status=204)
self.assertEqual(res.body, '')
)
from heka.holder import get_client
NO_API_KEY = {
"error": {
"errors": [{
"domain": "usageLimits",
"reason": "keyInvalid",
"message": "No API key was found",
}],
"code": 400,
"message": "No API key",
}
}
NO_API_KEY = dumps(NO_API_KEY)
NOT_FOUND = {
"error": {
"errors": [{
"domain": "geolocation",
"reason": "notFound",
"message": "Not found",
}],
"code": 404,
"message": "Not found",
}
}
NOT_FOUND = dumps(NOT_FOUND)
PARSE_ERROR = {
ApiKey
)
from ichnaea.decimaljson import dumps
NO_API_KEY = {
"error": {
"errors": [{
"domain": "usageLimits",
"reason": "keyInvalid",
"message": "No API key was found",
}],
"code": 400,
"message": "No API key",
}
}
NO_API_KEY = dumps(NO_API_KEY)
def check_api_key(func_name, error_on_invalidkey=False):
def c(func):
@wraps(func)
def closure(request, *args, **kwargs):
api_key = request.GET.get('key', None)
heka_client = get_heka_client()
if api_key is None:
heka_client.incr('%s.no_api_key' % func_name)
if error_on_invalidkey:
result = HTTPBadRequest()
result.content_type = 'application/json'
result.body = NO_API_KEY
)
NO_API_KEY = {
"error": {
"errors": [{
"domain": "usageLimits",
"reason": "keyInvalid",
"message": "No API key was found",
}],
"code":
400,
"message":
"No API key",
}
}
NO_API_KEY = dumps(NO_API_KEY)
NOT_FOUND = {
"error": {
"errors": [{
"domain": "geolocation",
"reason": "notFound",
"message": "Not found",
}],
"code":
404,
"message":
"Not found",
}
}
NOT_FOUND = dumps(NOT_FOUND)
search_wifi,
)
NOT_FOUND = {
"error": {
"errors": [{
"domain": "geolocation",
"reason": "notFound",
"message": "Not found",
}],
"code": 404,
"message": "Not found",
}
}
NOT_FOUND = dumps(NOT_FOUND)
PARSE_ERROR = {
"error": {
"errors": [{
"domain": "global",
"reason": "parseError",
"message": "Parse Error",
}],
"code": 400,
"message": "Parse Error"
}
}
PARSE_ERROR = dumps(PARSE_ERROR)
def __init__(self, errors, status=400):
body = {'errors': errors}
Response.__init__(self, dumps(body))
self.status = status
self.content_type = 'application/json'
ApiKey
)
from ichnaea.decimaljson import dumps
NO_API_KEY = {
"error": {
"errors": [{
"domain": "usageLimits",
"reason": "keyInvalid",
"message": "No API key was found",
}],
"code": 400,
"message": "No API key",
}
}
NO_API_KEY = dumps(NO_API_KEY)
def check_api_key(func_name, error_on_invalidkey=False):
def c(func):
@wraps(func)
def closure(request, *args, **kwargs):
api_key = request.GET.get('key', None)
heka_client = get_heka_client()
if api_key is None:
heka_client.incr('%s.no_api_key' % func_name)
if error_on_invalidkey:
result = HTTPBadRequest()
result.content_type = 'application/json'
result.body = NO_API_KEY
def __init__(self, errors, status=400):
body = {'errors': errors}
Response.__init__(self, dumps(body))
self.status = status
self.content_type = 'application/json'
logger = logging.getLogger('ichnaea')
NOT_FOUND = {
"error": {
"errors": [{
"domain": "geolocation",
"reason": "notFound",
"message": "Not found",
}],
"code":
404,
"message":
"Not found",
}
}
NOT_FOUND = dumps(NOT_FOUND)
PARSE_ERROR = {
"error": {
"errors": [{
"domain": "global",
"reason": "parseError",
"message": "Parse Error",
}],
"code":
400,
"message":
"Parse Error"
}
}
PARSE_ERROR = dumps(PARSE_ERROR)
