def uploadRRCInferenceSizes(self, **unused_args):
"""Handler for uploadRRCInferenceSizes request generated by client.
Takes the results of the RRC size depencence task and stores
them in the database.
Note that this is the result of *one* test, i.e. one sequence of
three packets sent. A set of tests with varying inter-packet
intervals can be identified by all having the same test_id."""
inferred_data = json.loads(self.request.body)
data = model.RRCInferenceSizesRawData()
# ID that uniquely identifies each phone, hashed
data.phone_id = util.HashDeviceId(\
str(inferred_data['phone_id']))
# Test ID that is unique for each device and set of tests
data.test_id = inferred_data['test_id']
data.network_type = inferred_data['network_type']
data.size = inferred_data['size']
data.result = inferred_data['result']
# The corresponding inter-packet interval
data.time_delay = inferred_data['time_delay']
#get the current time in utc
data.timestamp = datetime.datetime.utcnow()
data.put()
return
def testIMEIHash(self):
"""Test the hashing of device ID and extraction of type allocation code.
This test will fail if the the type allocation code is not preserved or
two different IMEIs produce the same hash.
"""
# Generate two random 15-digit numbers.
imei1 = str(random.randint(0, 1e16 - 1)).zfill(15)
imei2 = imei1
while imei1 == imei2:
imei2 = str(random.randint(0, 1e16 - 1)).zfill(15)
tac1 = util.GetTypeAllocationCode(imei1)
self.assertEqual(tac1, imei1[0:8], "Type allocation code mismatch")
tac2 = util.GetTypeAllocationCode(imei2)
self.assertEqual(tac2, imei2[0:8], "Type allocation code mismatch")
hash1 = util.HashDeviceId(imei1)
hash2 = util.HashDeviceId(imei2)
self.assertNotEqual(hash1, hash2, "Hash collision")
self.assertNotEqual(hash1, imei1[8:], "IMEI not hashed")
self.assertNotEqual(hash2, imei2[8:], "IMEI not hashed")
def uploadRRCInference(self, **unused_args):
"""Handler for uploadRRCInference request generated from client.
Take the results of the RRC inference tasks and store them
in the database.
Note that this is the result of *one* test, i.e. one sequence of
three packets sent. A set of tests with varying inter-packet
intervals can be identified by all having the same test_id."""
inferred_data = json.loads(self.request.body)
# read all the json param sent from the client.
raw_data = model.RRCInferenceRawData()
# ID that uniquely identifies each phone, hashed
raw_data.phone_id = util.HashDeviceId(\
str(inferred_data['phone_id']))
# Test ID that is unique for each device and set of tests
raw_data.test_id = inferred_data['test_id']
raw_data.network_type = inferred_data['network_type']
# Round trip times for the small packets
raw_data.rtt_low = inferred_data['rtt_low']
# Round trip times for the large packets
raw_data.rtt_high = inferred_data['rtt_high']
# number of packets lost for the small packets
raw_data.lost_low = inferred_data['lost_low']
# number of packets lost for the large packets
raw_data.lost_high = inferred_data['lost_high']
# signal strength at the time the small packets were sent
raw_data.signal_low = inferred_data['signal_low']
# signal strength at the time the large packets were sent
raw_data.signal_high = inferred_data['signal_high']
# Error values currently not implemented
raw_data.error_low = inferred_data['error_low']
raw_data.error_high = inferred_data['error_high']
# The corresponding inter-packet interval
raw_data.time_delay = inferred_data['time_delay']
#get the current time in utc
raw_data.timestamp = datetime.datetime.utcnow()
#write to DB/model
raw_data.put()
return
def PostMeasurement(self, **unused_args):
"""Handler used to post a measurement from a device."""
if self.request.method.lower() != 'post':
raise error.BadRequest('Not a POST request.')
try:
measurement_list = json.loads(self.request.body)
logging.info('PostMeasurement: Got %d measurements to write',
len(measurement_list))
for measurement_dict in measurement_list:
# Change device id such that it is anonymized, but preserve the TAC.
measurement_dict['tac'] = util.GetTypeAllocationCode(
measurement_dict['device_id'])
measurement_dict['device_id'] = util.HashDeviceId(
measurement_dict['device_id'])
device_info = model.DeviceInfo.get_or_insert(
measurement_dict['device_id'])
# Write new device properties, if present
if 'properties' in measurement_dict:
device_properties = model.DeviceProperties(
parent=device_info)
device_properties.device_info = device_info
properties_dict = measurement_dict['properties']
# TODO(wenjiezeng): Sanitize input that contains bad fields
util.ConvertFromDict(device_properties, properties_dict)
# Don't want the embedded properties in the Measurement object
del measurement_dict['properties']
else:
# Get most recent device properties
device_properties = device.GetLatestDeviceProperties(
device_info, create_new_if_none=True)
device_properties.put()
measurement = model.Measurement(parent=device_info)
util.ConvertFromDict(measurement, measurement_dict)
measurement.device_properties = device_properties
measurement.put()
except Exception, e:
logging.exception('Got exception posting measurements')
def Checkin(self, **unused_args):
"""Handler for checkin requests."""
if self.request.method.lower() != 'post':
raise error.BadRequest('Not a POST request.')
checkin = json.loads(self.request.body)
logging.info('Got checkin: %s', self.request.body)
try:
# Change device id such that it is anonymized, but preserve TAC.
checkin['tac'] = util.GetTypeAllocationCode(checkin['id'])
checkin['id'] = util.HashDeviceId(checkin['id'])
# Extract DeviceInfo.
device_id = checkin['id']
logging.info('Checkin from device %s', device_id)
device_info = model.DeviceInfo.get_or_insert(device_id)
device_info.user = users.get_current_user()
# Don't want the embedded properties in the device_info structure.
device_info_dict = dict(checkin)
del device_info_dict['properties']
util.ConvertFromDict(device_info, device_info_dict)
device_info.put()
# Extract DeviceProperties.
device_properties = model.DeviceProperties(parent=device_info)
device_properties.device_info = device_info
util.ConvertFromDict(device_properties, checkin['properties'])
device_properties.put()
device_schedule = GetDeviceSchedule(device_properties)
device_schedule_json = EncodeScheduleAsJson(device_schedule)
logging.info('Sending checkin response: %s', device_schedule_json)
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(device_schedule_json)
except Exception, e:
logging.error('Got exception during checkin: %s', e)
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps([]))
def PostMeasurement(self, **unused_args):
"""Handler used to post a measurement from a device."""
if self.request.method.lower() != 'post':
raise error.BadRequest('Not a POST request.')
try:
measurement_list = json.loads(self.request.body)
logging.info('PostMeasurement: Got %d measurements to write',
len(measurement_list))
for measurement_dict in measurement_list:
# Change device id such that it is anonymized, but preserve the TAC.
measurement_dict['tac'] = util.GetTypeAllocationCode(
measurement_dict['device_id'])
measurement_dict['device_id'] = util.HashDeviceId(
measurement_dict['device_id'])
device_info = model.DeviceInfo.get_or_insert(
measurement_dict['device_id'])
# Write new device properties, if present
if 'properties' in measurement_dict:
device_properties = model.DeviceProperties(
parent=device_info)
device_properties.device_info = device_info
properties_dict = measurement_dict['properties']
# TODO(wenjiezeng): Sanitize input that contains bad fields
util.ConvertFromDict(device_properties, properties_dict)
# Don't want the embedded properties in the Measurement object
del measurement_dict['properties']
else:
# Get most recent device properties
device_properties = device.GetLatestDeviceProperties(
device_info, create_new_if_none=True)
device_properties.put()
if 'context_results' in measurement_dict:
if 'values' in measurement_dict:
measurement_dict['values']['context_results'] = str(
measurement_dict['context_results'])
del measurement_dict['context_results']
measurement = model.Measurement(parent=device_info)
util.ConvertFromDict(measurement, measurement_dict)
measurement.device_properties = device_properties
measurement.put()
#extracting the IPs from the ping measurement results to the main CDN domain
if measurement.success == True and measurement.type == "ping":
if ('target' in measurement_dict['parameters']
) and measurement_dict['parameters'][
'target'] in CDN_TARGETS:
ipdata = model.CDNIpData()
target_ip = measurement_dict['values'][
'target_ip'].replace('"', '')
if ':' in target_ip:
continue
prefix = target_ip[:target_ip.rfind('.') + 1].replace(
'"', '')
q = model.CDNIpData.all()
q.filter("prefix =", prefix)
record = q.get()
if record != None:
#updating timestamp (expiration time)
record.put()
else:
#inserting the new IP record to the data store
record = model.CDNIpData()
record.ip = target_ip
record.prefix = prefix
record.cdn_domain = measurement_dict['parameters'][
'target']
record.put()
nettype = properties_dict['network_type'].lower().replace(
" ", "")
# map from deviceid to the list of high ping rtts
low_performance_devices = {}
if measurement.success == True and measurement.type == "ping" and nettype != "wifi":
if ('target' in measurement_dict['parameters']
) and measurement_dict['parameters'][
'target'] in CDN_TARGETS:
mean_rtt = float(
measurement_dict['values']['mean_rtt_ms'])
if util.IsLowPerformance("ping", nettype, mean_rtt):
lat = float(
properties_dict['location']['latitude'])
lon = float(
properties_dict['location']['longitude'])
carrier = properties_dict['carrier'].lower(
).replace(" ", "")
timestamp = measurement_dict['timestamp']
target = measurement_dict['parameters']['target']
# get all the measurement results from the last 24 hours
q = model.Measurement.all()
q.filter("type =", "ping")
q.filter("success =", True)
q.filter("timestamp >",
datetime.now() - timedelta(hours=24))
for record in q.run():
record_nettype = record.device_properties.network_type.lower(
).replace(" ", "")
if record.GetParam(
'target'
) == target and record_nettype != "wifi":
record_rtt = record.GetValue('mean_rtt_ms')
if util.IsLowPerformance(
"ping", record_nettype,
record_rtt):
record_location = record.device_properties.location
record_carrier = record.device_properties.carrier
distance = -1
if record_location.lat != 0.0 and record_location.lon != 0.0:
distance = util.Distance(
lat, lon, record_location.lat,
record_location.lon)
if (
distance != -1
and distance < 2000
) or record_carrier == carrier: #distance in km
if not (record.device_properties.
device_info.id in
low_performance_devices):
low_performance_devices[
record.device_properties.
device_info.id] = []
low_performance_devices[
record.device_properties.
device_info.id].append(
record_rtt)
#the number of devices should be at-least two
if len(low_performance_devices.keys()) >= 2:
query = model.Task.all()
query.filter(
"tag = ", "DIAG_TARGET"
) #specific ping tasks for target diagnosis, for example, ping to www.facebook.com
for diag_target_task in query.run():
filter = diag_target_task.filter
if not (str(device_info.id) in filter):
if filter == None:
diag_target_task.filter = "id = " + str(
device_info.id)
else:
diag_target_task.filter = diag_target_task.filter + " OR id = " + str(
device_info.id)
diag_target_task.put()
except Exception, e:
logging.exception('Got exception posting measurements')