def TimeseriesData(self, **unused_args):
"""Returns data for the timeseries view in JSON format."""
device_id = self.request.get('device_id')
start_time = self.request.get('start_time')
end_time = self.request.get('start_time')
limit = self.request.get('limit')
# Used to trigger permission check
unused_device = model.DeviceInfo.GetDeviceWithAcl(device_id)
if start_time:
start_time = util.MicrosecondsSinceEpochToTime(int(start_time))
if end_time:
end_time = util.MicrosecondsSinceEpochToTime(int(end_time))
if limit:
limit = int(limit)
measurements = model.Measurement.GetMeasurementListWithAcl(
limit=limit,
device_id=device_id,
start_time=start_time,
end_time=end_time)
tsdata = []
for meas in measurements:
ms_time = util.TimeToMicrosecondsSinceEpoch(meas.timestamp) / 1000
rssi = meas.device_properties.rssi or 0
battery = meas.device_properties.battery_level or 0
val = ('new Date(%d)' % ms_time, rssi, battery)
tsdata.append(val)
self.response.out.write(json.dumps(tsdata))
def TimeseriesData(self, **unused_args):
"""Returns data for the timeseries view in JSON format."""
start_time = self.request.get('start_time')
end_time = self.request.get('start_time')
limit = self.request.get('limit')
sample_type = self.request.get('type')
summaries = model.ValidationSummary.all()
if start_time:
start_time = util.MicrosecondsSinceEpochToTime(int(start_time))
summaries.filter('timestamp_start > ', start_time)
if end_time:
end_time = util.MicrosecondsSinceEpochToTime(int(end_time))
summaries.filter('timestamp_end < ', end_time)
if limit:
limit = int(limit)
else:
limit = 1000
summaries.filter('timestamp_start > ', 0)
summaries.order('timestamp_start')
time_to_type_to_count = dict()
tsdata = []
# group data by timestamp for timeline printing
for summary in summaries.fetch(limit):
ms_time = util.TimeToMicrosecondsSinceEpoch(
summary.timestamp_start) / 1e3
if not time_to_type_to_count.has_key(ms_time):
time_to_type_to_count[ms_time] = dict()
if sample_type:
time_to_type_to_count[ms_time][summary.measurement_type] = \
getattr(summary, sample_type)
else: time_to_type_to_count[ms_time][summary.measurement_type] = \
summary.record_count
# gather data into timeline-friendly output format
for time, type_to_count in time_to_type_to_count.items():
row = ['new Date(%d)' % time]
for meas, name in measurement.MEASUREMENT_TYPES:
if type_to_count.has_key(meas):
row.append(type_to_count[meas])
else:
row.append(0)
tsdata.append(row)
self.response.out.write(json.dumps(tsdata))
def Validate(self, **unused_args):
"""Main handler for the validation view.
Note that this method is called from a request that is restricted to
admin privileges.
Args (HTTP request parameters):
start_time: start time for validation period (ISO8601 format)
end_time: end time for validation period (ISO8601 format)
iters: (optional) Number of days to do bulk validation on. (int)
use_webpage: (optional) Show validation results in HTML. If set to false,
send an e-mail with results. (boolean)
worker: (optional) True if running as a queued task. (boolean)
Returns:
Result of validation, or nothing.
Raises:
No exceptions handled here.
No new exceptions generated here.
"""
iters = self.request.get('iters')
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
# if iterating, set up and enqueue validation tasks
while iters and int(iters) > 1:
start_time = 24 * 60 * 60 * 1000 * 1000 + \
util.TimeToMicrosecondsSinceEpoch(util.StringToTime(start_time))
start_time = util.MicrosecondsSinceEpochToTime(start_time)
start_time = util.TimeToString(start_time)
end_time = 24 * 60 * 60 * 1000 * 1000 + \
util.TimeToMicrosecondsSinceEpoch(util.StringToTime(end_time))
end_time = util.MicrosecondsSinceEpochToTime(end_time)
end_time = util.TimeToString(end_time)
# Add the task to the 'validation' queue.
taskqueue.add(
url='/validation/data?worker=true&start_time=%s&end_time=%s' %
(start_time, end_time),
method='GET',
queue_name='validation')
iters = int(iters) - 1
# return here if iterating using task queue
if iters:
self.response.out.write("{Success:true}")
return
# contains validation results for printing
self.validation_results = dict()
# support only the measurements specified in MEASUREMENT_TYPES
for mtype, name in measurement.MEASUREMENT_TYPES:
self.type_to_summary[mtype] = \
model.ValidationSummary(measurement_type=mtype)
self.type_to_details[mtype] = []
# validate all the data in one pass
self._DoValidation()
# validation results are in type_to_details, now write them to datastore
for mtype, data in self.type_to_summary.items():
data.put(
) # must put summary before putting details that reference them
if self.type_to_details.has_key(mtype):
for detail in self.type_to_details[mtype]:
detail.summary = data
detail.put()
# if this was a queued task, return here
if self.request.get('worker'):
self.response.out.write("{Success:true}")
return
# for display purposes, render HTML of results
html = template.render('templates/validation.html',
self.validation_results)
# send to response, or e-mail user
if self.request.get('use_webpage'):
self.response.out.write(html)
else:
message = mail.EmailMessage(sender=config.VALIDATION_EMAIL_SENDER,
subject="Daily validation results")
message.to = config.VALIDATION_EMAIL_RECIPIENT
message.body = html
message.html = html
message.send()
def _DoValidation(self):
"""Gets all the records for a specified type, subject
to request parameters.
Args:
None. This reads from the measurement request
Returns:
Nothing. It writes results to an instance variable.
Raises:
No exceptions handled here.
No new exceptions generated here.
"""
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
limit = self.request.get('limit')
# manually specified parameters for testing
if self.TESTING:
limit = 100
start_time = util.TimeToMicrosecondsSinceEpoch(
util.StringToTime("2012-03-20T00:00:00Z"))
end_time = util.TimeToMicrosecondsSinceEpoch(
util.StringToTime("2012-03-21T00:00:00Z"))
# set up query filters according to parameters
query = model.Measurement.all()
if start_time:
dt_start = util.TimeToMicrosecondsSinceEpoch(
util.StringToTime(start_time))
else:
dt_start = util.TimeToMicrosecondsSinceEpoch(
datetime.datetime.utcfromtimestamp(time.time()) -
datetime.timedelta(days=1))
self.validation_results['start_time'] = \
util.MicrosecondsSinceEpochToTime(dt_start)
query.filter('timestamp >=', dt_start)
if end_time:
dt = util.TimeToMicrosecondsSinceEpoch(util.StringToTime(end_time))
else:
dt = util.TimeToMicrosecondsSinceEpoch(
datetime.datetime.utcfromtimestamp(time.time()))
self.validation_results[
'end_time'] = util.MicrosecondsSinceEpochToTime(dt)
query.filter('timestamp <', dt)
query.order('timestamp')
if limit:
results = query.fetch(int(limit))
else:
results = query
error_type_to_count = dict() # map of error type to count
num_invalid = dict() # map of measurement type to count of invalid
for measurement in results:
# Need to catch case where device has been deleted
try:
unused_device_info = measurement.device_properties.device_info
except db.ReferencePropertyResolveError:
logging.exception('Device deleted for measurement %s',
measurement.key().id())
# Skip this measurement
continue
# catch case where measurement is not supported
if not self.type_to_summary.has_key(measurement.type):
self.type_to_summary[measurement.type] = \
model.ValidationSummary(measurement_type=measurement.type,
error="UnknownType")
# set initial data for validation summary
if not self.type_to_summary[measurement.type].timestamp_start:
self.type_to_summary[measurement.type].timestamp_start = \
util.MicrosecondsSinceEpochToTime(dt_start)
self.type_to_summary[measurement.type].timestamp_end = \
util.MicrosecondsSinceEpochToTime(dt)
self.type_to_summary[measurement.type].record_count = 1
self.type_to_summary[measurement.type].error_count = 0
num_invalid[measurement.type] = 0
else:
self.type_to_summary[measurement.type].record_count += 1
# Make a measurement-specific validation object
try:
validator = MeasurementValidatorFactory.CreateValidator(
measurement)
except RuntimeError:
continue # no validator defined, continue
valid = validator.Validate()
if not valid['valid']:
if not num_invalid.has_key(measurement.type):
num_invalid[measurement.type] = 0
num_invalid[measurement.type] += 1
# print the error details
if self.PRINT_VALUES or (self.PRINT_INVALID
and not valid['valid']):
if not self.validation_results.has_key(
'%s_invalid_detail' % measurement.type):
self.validation_results['%s_invalid_detail' %
measurement.type] = []
self.validation_results[
'%s_invalid_detail' % measurement.type].append(
'Errors: %s <br>\nTime: %s<br>\nDevice: %s<br>\nDetails: %s'
% (", ".join(
valid['error_types']), str(measurement.timestamp),
measurement.device_properties.device_info.id,
validator.PrintData()))
# update error counters
for error in valid['error_types']:
if not error_type_to_count.has_key(measurement.type):
error_type_to_count[measurement.type] = dict()
if not error_type_to_count[measurement.type].has_key(
error):
error_type_to_count[measurement.type][error] = 1
else:
error_type_to_count[measurement.type][error] += 1
# create ValidationEntry for each error
if len(valid['error_types']) > 0:
error_detail = model.ValidationEntry()
#error_detail.summary = self.type_to_summary[measurement.type]
error_detail.measurement = measurement
error_detail.error_types = valid['error_types']
self.type_to_details[measurement.type].append(error_detail)
# update number of invalid measurements
self.validation_results['%s_invalid' % measurement.type] = \
num_invalid[measurement.type]
# update validation summary entity for each data type
for data_type in self.type_to_summary.keys():
if num_invalid.has_key(data_type):
self.type_to_summary[data_type].error_count = num_invalid[
data_type]
if error_type_to_count.has_key(data_type):
self.type_to_summary[data_type].SetErrorByType(
error_type_to_count[data_type])
self.validation_results[data_type + "_count"] = \
self.type_to_summary[data_type].record_count