def testIMEIHash(self):
"""Test the hashing of device ID and extraction of type allocation code.
This test will fail if the the type allocation code is not preserved or
two different IMEIs produce the same hash.
"""
# Generate two random 15-digit numbers.
imei1 = str(random.randint(0, 1e16 - 1)).zfill(15)
imei2 = imei1
while imei1 == imei2:
imei2 = str(random.randint(0, 1e16 - 1)).zfill(15)
tac1 = util.GetTypeAllocationCode(imei1)
self.assertEqual(tac1, imei1[0:8], "Type allocation code mismatch")
tac2 = util.GetTypeAllocationCode(imei2)
self.assertEqual(tac2, imei2[0:8], "Type allocation code mismatch")
hash1 = util.HashDeviceId(imei1)
hash2 = util.HashDeviceId(imei2)
self.assertNotEqual(hash1, hash2, "Hash collision")
self.assertNotEqual(hash1, imei1[8:], "IMEI not hashed")
self.assertNotEqual(hash2, imei2[8:], "IMEI not hashed")
def Checkin(self, **unused_args):
"""Handler for checkin requests."""
if self.request.method.lower() != 'post':
raise error.BadRequest('Not a POST request.')
checkin = json.loads(self.request.body)
logging.info('Got checkin: %s', self.request.body)
try:
# Change device id such that it is anonymized, but preserve TAC.
checkin['tac'] = util.GetTypeAllocationCode(checkin['id'])
# We need the actual id, because we need to make a decision which ue need to take the task.
# checkin['id'] = util.HashDeviceId(checkin['id'])
# Extract DeviceInfo.
device_id = checkin['id']
logging.info('Checkin from device %s', device_id)
device_info = model.DeviceInfo.get_or_insert(device_id)
device_info.user = users.get_current_user()
# Don't want the embedded properties in the device_info structure.
device_info_dict = dict(checkin)
del device_info_dict['properties']
util.ConvertFromDict(device_info, device_info_dict)
device_info.put()
# Extract DeviceProperties.
device_properties = model.DeviceProperties(parent=device_info)
device_properties.device_info = device_info
util.ConvertFromDict(device_properties, checkin['properties'])
device_properties.put()
device_schedule = GetDeviceSchedule(device_properties)
device_schedule_json = EncodeScheduleAsJson(device_schedule)
logging.info('Sending checkin response: %s', device_schedule_json)
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(device_schedule_json)
except Exception, e:
logging.error('Got exception during checkin: %s', e)
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps([]))
def PostMeasurement(self, **unused_args):
"""Handler used to post a measurement from a device."""
if self.request.method.lower() != 'post':
raise error.BadRequest('Not a POST request.')
try:
measurement_list = json.loads(self.request.body)
logging.info('PostMeasurement: Got %d measurements to write',
len(measurement_list))
for measurement_dict in measurement_list:
# Change device id such that it is anonymized, but preserve the TAC.
measurement_dict['tac'] = util.GetTypeAllocationCode(
measurement_dict['device_id'])
measurement_dict['device_id'] = util.HashDeviceId(
measurement_dict['device_id'])
device_info = model.DeviceInfo.get_or_insert(
measurement_dict['device_id'])
# Write new device properties, if present
if 'properties' in measurement_dict:
device_properties = model.DeviceProperties(
parent=device_info)
device_properties.device_info = device_info
properties_dict = measurement_dict['properties']
# TODO(wenjiezeng): Sanitize input that contains bad fields
util.ConvertFromDict(device_properties, properties_dict)
# Don't want the embedded properties in the Measurement object
del measurement_dict['properties']
else:
# Get most recent device properties
device_properties = device.GetLatestDeviceProperties(
device_info, create_new_if_none=True)
device_properties.put()
measurement = model.Measurement(parent=device_info)
util.ConvertFromDict(measurement, measurement_dict)
measurement.device_properties = device_properties
measurement.put()
except Exception, e:
logging.exception('Got exception posting measurements')
def PostMeasurement(self, **unused_args):
"""Handler used to post a measurement from a device."""
if self.request.method.lower() != 'post':
raise error.BadRequest('Not a POST request.')
try:
measurement_list = json.loads(self.request.body)
logging.info('PostMeasurement: Got %d measurements to write',
len(measurement_list))
for measurement_dict in measurement_list:
# Change device id such that it is anonymized, but preserve the TAC.
measurement_dict['tac'] = util.GetTypeAllocationCode(
measurement_dict['device_id'])
measurement_dict['device_id'] = util.HashDeviceId(
measurement_dict['device_id'])
device_info = model.DeviceInfo.get_or_insert(
measurement_dict['device_id'])
# Write new device properties, if present
if 'properties' in measurement_dict:
device_properties = model.DeviceProperties(
parent=device_info)
device_properties.device_info = device_info
properties_dict = measurement_dict['properties']
# TODO(wenjiezeng): Sanitize input that contains bad fields
util.ConvertFromDict(device_properties, properties_dict)
# Don't want the embedded properties in the Measurement object
del measurement_dict['properties']
else:
# Get most recent device properties
device_properties = device.GetLatestDeviceProperties(
device_info, create_new_if_none=True)
device_properties.put()
if 'context_results' in measurement_dict:
if 'values' in measurement_dict:
measurement_dict['values']['context_results'] = str(
measurement_dict['context_results'])
del measurement_dict['context_results']
measurement = model.Measurement(parent=device_info)
util.ConvertFromDict(measurement, measurement_dict)
measurement.device_properties = device_properties
measurement.put()
#extracting the IPs from the ping measurement results to the main CDN domain
if measurement.success == True and measurement.type == "ping":
if ('target' in measurement_dict['parameters']
) and measurement_dict['parameters'][
'target'] in CDN_TARGETS:
ipdata = model.CDNIpData()
target_ip = measurement_dict['values'][
'target_ip'].replace('"', '')
if ':' in target_ip:
continue
prefix = target_ip[:target_ip.rfind('.') + 1].replace(
'"', '')
q = model.CDNIpData.all()
q.filter("prefix =", prefix)
record = q.get()
if record != None:
#updating timestamp (expiration time)
record.put()
else:
#inserting the new IP record to the data store
record = model.CDNIpData()
record.ip = target_ip
record.prefix = prefix
record.cdn_domain = measurement_dict['parameters'][
'target']
record.put()
nettype = properties_dict['network_type'].lower().replace(
" ", "")
# map from deviceid to the list of high ping rtts
low_performance_devices = {}
if measurement.success == True and measurement.type == "ping" and nettype != "wifi":
if ('target' in measurement_dict['parameters']
) and measurement_dict['parameters'][
'target'] in CDN_TARGETS:
mean_rtt = float(
measurement_dict['values']['mean_rtt_ms'])
if util.IsLowPerformance("ping", nettype, mean_rtt):
lat = float(
properties_dict['location']['latitude'])
lon = float(
properties_dict['location']['longitude'])
carrier = properties_dict['carrier'].lower(
).replace(" ", "")
timestamp = measurement_dict['timestamp']
target = measurement_dict['parameters']['target']
# get all the measurement results from the last 24 hours
q = model.Measurement.all()
q.filter("type =", "ping")
q.filter("success =", True)
q.filter("timestamp >",
datetime.now() - timedelta(hours=24))
for record in q.run():
record_nettype = record.device_properties.network_type.lower(
).replace(" ", "")
if record.GetParam(
'target'
) == target and record_nettype != "wifi":
record_rtt = record.GetValue('mean_rtt_ms')
if util.IsLowPerformance(
"ping", record_nettype,
record_rtt):
record_location = record.device_properties.location
record_carrier = record.device_properties.carrier
distance = -1
if record_location.lat != 0.0 and record_location.lon != 0.0:
distance = util.Distance(
lat, lon, record_location.lat,
record_location.lon)
if (
distance != -1
and distance < 2000
) or record_carrier == carrier: #distance in km
if not (record.device_properties.
device_info.id in
low_performance_devices):
low_performance_devices[
record.device_properties.
device_info.id] = []
low_performance_devices[
record.device_properties.
device_info.id].append(
record_rtt)
#the number of devices should be at-least two
if len(low_performance_devices.keys()) >= 2:
query = model.Task.all()
query.filter(
"tag = ", "DIAG_TARGET"
) #specific ping tasks for target diagnosis, for example, ping to www.facebook.com
for diag_target_task in query.run():
filter = diag_target_task.filter
if not (str(device_info.id) in filter):
if filter == None:
diag_target_task.filter = "id = " + str(
device_info.id)
else:
diag_target_task.filter = diag_target_task.filter + " OR id = " + str(
device_info.id)
diag_target_task.put()
except Exception, e:
logging.exception('Got exception posting measurements')