def ListMeasurements(self, **unused_args):
"""Handles /measurements REST request."""
# This is very limited and only supports time range and limit
# queries. You might want to extend this to filter by other
# properties, but for the purpose of measurement archiving
# this is all we need.
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
limit = self.request.get('limit')
query = model.Measurement.all()
if start_time:
dt = util.MicrosecondsSinceEpochToTime(int(start_time))
query.filter('timestamp >=', dt)
if end_time:
dt = util.MicrosecondsSinceEpochToTime(int(end_time))
query.filter('timestamp <', dt)
query.order('timestamp')
if limit:
results = query.fetch(int(limit))
else:
results = query
output = util.MeasurementListToDictList(results)
self.response.out.write(json.dumps(output))
def CommonExceptions(self, **unused_args):
"""Returns a list containing the most common 10 exceptions and the number
of times they have been reported."""
start_time = self.request.get('start_time')
end_time = self.request.get('start_time')
limit = self.request.get('limit')
entries = model.ValidationEntry.all()
logging.log(logging.INFO, "Found %s records" %
model.ValidationEntry.all().count(10000))
# TODO(drc): support date queries
if start_time:
start_time = util.MicrosecondsSinceEpochToTime(int(start_time))
if end_time:
end_time = util.MicrosecondsSinceEpochToTime(int(end_time))
if limit:
limit = int(limit)
else: limit = 1000
error_to_count = dict()
for entry in entries.fetch(limit):
if entry.measurement.success == False:
# Only grab first portion of stack trace if there was an error
if not (entry.measurement.GetValue('error') is None):
error = "<br>".join(
entry.measurement.GetValue('error').split("\n")[0:5])
if not error_to_count.has_key(error):
error_to_count[error] = 1
else: error_to_count[error] += 1
sorted_errors = sorted(error_to_count.iteritems(),
key=operator.itemgetter(1), reverse=True)
return sorted_errors[0:10]
def TimeseriesData(self, **unused_args):
"""Returns data for the timeseries view in JSON format."""
device_id = self.request.get('device_id')
start_time = self.request.get('start_time')
end_time = self.request.get('start_time')
limit = self.request.get('limit')
# Used to trigger permission check
unused_device = model.DeviceInfo.GetDeviceWithAcl(device_id)
if start_time:
start_time = util.MicrosecondsSinceEpochToTime(int(start_time))
if end_time:
end_time = util.MicrosecondsSinceEpochToTime(int(end_time))
if limit:
limit = int(limit)
measurements = model.Measurement.GetMeasurementListWithAcl(
limit=limit,
device_id=device_id,
start_time=start_time,
end_time=end_time)
tsdata = []
for meas in measurements:
ms_time = util.TimeToMicrosecondsSinceEpoch(meas.timestamp) / 1000
rssi = meas.device_properties.rssi or 0
battery = meas.device_properties.battery_level or 0
val = ('new Date(%d)' % ms_time, rssi, battery)
tsdata.append(val)
self.response.out.write(json.dumps(tsdata))
def ListMeasurements(self, **unused_args):
"""Handles /measurements REST request."""
# This is very limited and only supports time range and limit
# queries. You might want to extend this to filter by other
# properties, but for the purpose of measurement archiving
# this is all we need.
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
limit = self.request.get('limit')
query = model.Measurement.all()
if start_time:
dt = util.MicrosecondsSinceEpochToTime(int(start_time))
query.filter('timestamp >=', dt)
if end_time:
dt = util.MicrosecondsSinceEpochToTime(int(end_time))
query.filter('timestamp <', dt)
query.order('timestamp')
if limit:
results = query.fetch(int(limit))
else:
results = query
output = []
for measurement in results:
# Need to catch case where device has been deleted
try:
unused_device_info = measurement.device_properties.device_info
except db.ReferencePropertyResolveError:
logging.exception('Device deleted for measurement %s',
measurement.key().id())
# Skip this measurement
continue
# Need to catch case where task has been deleted
try:
unused_task = measurement.task
except db.ReferencePropertyResolveError:
measurement.task = None
measurement.put()
mdict = util.ConvertToDict(measurement,
timestamps_in_microseconds=True)
# Fill in additional fields
mdict['id'] = str(measurement.key().id())
mdict['parameters'] = measurement.Params()
mdict['values'] = measurement.Values()
if 'task' in mdict and mdict['task'] is not None:
mdict['task']['id'] = str(measurement.GetTaskID())
mdict['task']['parameters'] = measurement.task.Params()
output.append(mdict)
self.response.out.write(json.dumps(output))
def _Archive(self, **unused_args):
"""Processes parameters and packages the data into a file-like object.
This method does the bulk of the heavy lifting. It makes sense of the
request parameters, serializes the data, generates a name used to
encapsulate the result, and makes the call to compress the file-like
object.
URL Args:
device_id: A string key for a device in the datastore.
start_time: The timestamp for the earliest measurement (microseconds UTC)
end_time: The timestamp for the latest measurement (microseconds UTC)
All of the URL Args are optional and in the case where none are given no
restrictions are assumed and all the data is returned. Since these
handlers should never be called by users it should be understood that this
case should be avoided.
Returns:
A name that is suitable for describing the contents of the file based on
the parameters for the request, and a file-like object containing the
measurement data.
Raises:
No exceptions handled here.
No new exceptions generated here.
"""
#TODO(mdw) Unit test needed.
# Make sense of parameters
device_id = self.request.get('device_id')
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
if start_time:
start = util.MicrosecondsSinceEpochToTime(int(start_time))
else:
start = None
if end_time:
end = util.MicrosecondsSinceEpochToTime(int(end_time))
else:
end = None
# Get data based on parameters
model_list = GetMeasurementDictList(device_id, start, end)
# Serialize the data
data = {'Measurement': json.dumps(model_list)}
# Generate directory/file name based on parameters
archive_dir = ParametersToFileNameBase(device_id, start_time, end_time)
# For some reason there was a problem with Unicode chars in the request
archive_dir = archive_dir.encode('ascii', 'ignore')
#NOTE: This is a multiple return.
return archive_dir, archive_util.ArchiveCompress(data,
directory=archive_dir)
def TimeseriesData(self, **unused_args):
"""Returns data for the timeseries view in JSON format."""
start_time = self.request.get('start_time')
end_time = self.request.get('start_time')
limit = self.request.get('limit')
sample_type = self.request.get('type')
summaries = model.ValidationSummary.all()
if start_time:
start_time = util.MicrosecondsSinceEpochToTime(int(start_time))
summaries.filter('timestamp_start > ', start_time)
if end_time:
end_time = util.MicrosecondsSinceEpochToTime(int(end_time))
summaries.filter('timestamp_end < ', end_time)
if limit:
limit = int(limit)
else:
limit = 1000
summaries.filter('timestamp_start > ', 0)
summaries.order('timestamp_start')
time_to_type_to_count = dict()
tsdata = []
# group data by timestamp for timeline printing
for summary in summaries.fetch(limit):
ms_time = util.TimeToMicrosecondsSinceEpoch(
summary.timestamp_start) / 1e3
if not time_to_type_to_count.has_key(ms_time):
time_to_type_to_count[ms_time] = dict()
if sample_type:
time_to_type_to_count[ms_time][summary.measurement_type] = \
getattr(summary, sample_type)
else: time_to_type_to_count[ms_time][summary.measurement_type] = \
summary.record_count
# gather data into timeline-friendly output format
for time, type_to_count in time_to_type_to_count.items():
row = ['new Date(%d)' % time]
for meas, name in measurement.MEASUREMENT_TYPES:
if type_to_count.has_key(meas):
row.append(type_to_count[meas])
else:
row.append(0)
tsdata.append(row)
self.response.out.write(json.dumps(tsdata))
def JSON_DECODE_timestamp(self, inputval):
try:
self.timestamp = util.MicrosecondsSinceEpochToTime(int(inputval))
except ValueError:
logging.exception('Error occurs while converting timestamp '
'%s to integer', inputval)
self.timestamp = None
def DashboardDetail(self, **unused_args):
"""Returns a dict of time to measurement type to count."""
start_time = self.request.get('start_time')
end_time = self.request.get('start_time')
limit = self.request.get('limit')
entries = model.ValidationEntry.all()
if start_time:
start_time = util.MicrosecondsSinceEpochToTime(int(start_time))
if end_time:
end_time = util.MicrosecondsSinceEpochToTime(int(end_time))
if limit:
limit = int(limit)
else:
limit = 1000
# TODO(drc): Incorporate date limits.
time_to_type_to_cnt = SortedDict()
# group by time
for ent in entries.fetch(limit):
ms_time = ent.summary.timestamp_start
meas_type = ent.summary.measurement_type
time_to_type_to_cnt.setdefault(ms_time, dict()).setdefault(
meas_type, {'details': []})
time_to_type_to_cnt[ms_time][meas_type][
'count'] = ent.summary.error_count
# links to ids for eventually showing more detail
time_to_type_to_cnt[ms_time][meas_type]['details'].append([
ent.measurement.key().id(),
ent.measurement.device_properties.device_info.id
])
# now sort by time
sorted_results = SortedDict()
for k in sorted(time_to_type_to_cnt.iterkeys()):
sorted_results[k] = time_to_type_to_cnt[k]
return sorted_results
def Validate(self, **unused_args):
"""Main handler for the validation view.
Note that this method is called from a request that is restricted to
admin privileges.
Args (HTTP request parameters):
start_time: start time for validation period (ISO8601 format)
end_time: end time for validation period (ISO8601 format)
iters: (optional) Number of days to do bulk validation on. (int)
use_webpage: (optional) Show validation results in HTML. If set to false,
send an e-mail with results. (boolean)
worker: (optional) True if running as a queued task. (boolean)
Returns:
Result of validation, or nothing.
Raises:
No exceptions handled here.
No new exceptions generated here.
"""
iters = self.request.get('iters')
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
# if iterating, set up and enqueue validation tasks
while iters and int(iters) > 1:
start_time = 24 * 60 * 60 * 1000 * 1000 + \
util.TimeToMicrosecondsSinceEpoch(util.StringToTime(start_time))
start_time = util.MicrosecondsSinceEpochToTime(start_time)
start_time = util.TimeToString(start_time)
end_time = 24 * 60 * 60 * 1000 * 1000 + \
util.TimeToMicrosecondsSinceEpoch(util.StringToTime(end_time))
end_time = util.MicrosecondsSinceEpochToTime(end_time)
end_time = util.TimeToString(end_time)
# Add the task to the 'validation' queue.
taskqueue.add(
url='/validation/data?worker=true&start_time=%s&end_time=%s' %
(start_time, end_time),
method='GET',
queue_name='validation')
iters = int(iters) - 1
# return here if iterating using task queue
if iters:
self.response.out.write("{Success:true}")
return
# contains validation results for printing
self.validation_results = dict()
# support only the measurements specified in MEASUREMENT_TYPES
for mtype, name in measurement.MEASUREMENT_TYPES:
self.type_to_summary[mtype] = \
model.ValidationSummary(measurement_type=mtype)
self.type_to_details[mtype] = []
# validate all the data in one pass
self._DoValidation()
# validation results are in type_to_details, now write them to datastore
for mtype, data in self.type_to_summary.items():
data.put(
) # must put summary before putting details that reference them
if self.type_to_details.has_key(mtype):
for detail in self.type_to_details[mtype]:
detail.summary = data
detail.put()
# if this was a queued task, return here
if self.request.get('worker'):
self.response.out.write("{Success:true}")
return
# for display purposes, render HTML of results
html = template.render('templates/validation.html',
self.validation_results)
# send to response, or e-mail user
if self.request.get('use_webpage'):
self.response.out.write(html)
else:
message = mail.EmailMessage(sender=config.VALIDATION_EMAIL_SENDER,
subject="Daily validation results")
message.to = config.VALIDATION_EMAIL_RECIPIENT
message.body = html
message.html = html
message.send()
def _DoValidation(self):
"""Gets all the records for a specified type, subject
to request parameters.
Args:
None. This reads from the measurement request
Returns:
Nothing. It writes results to an instance variable.
Raises:
No exceptions handled here.
No new exceptions generated here.
"""
start_time = self.request.get('start_time')
end_time = self.request.get('end_time')
limit = self.request.get('limit')
# manually specified parameters for testing
if self.TESTING:
limit = 100
start_time = util.TimeToMicrosecondsSinceEpoch(
util.StringToTime("2012-03-20T00:00:00Z"))
end_time = util.TimeToMicrosecondsSinceEpoch(
util.StringToTime("2012-03-21T00:00:00Z"))
# set up query filters according to parameters
query = model.Measurement.all()
if start_time:
dt_start = util.TimeToMicrosecondsSinceEpoch(
util.StringToTime(start_time))
else:
dt_start = util.TimeToMicrosecondsSinceEpoch(
datetime.datetime.utcfromtimestamp(time.time()) -
datetime.timedelta(days=1))
self.validation_results['start_time'] = \
util.MicrosecondsSinceEpochToTime(dt_start)
query.filter('timestamp >=', dt_start)
if end_time:
dt = util.TimeToMicrosecondsSinceEpoch(util.StringToTime(end_time))
else:
dt = util.TimeToMicrosecondsSinceEpoch(
datetime.datetime.utcfromtimestamp(time.time()))
self.validation_results[
'end_time'] = util.MicrosecondsSinceEpochToTime(dt)
query.filter('timestamp <', dt)
query.order('timestamp')
if limit:
results = query.fetch(int(limit))
else:
results = query
error_type_to_count = dict() # map of error type to count
num_invalid = dict() # map of measurement type to count of invalid
for measurement in results:
# Need to catch case where device has been deleted
try:
unused_device_info = measurement.device_properties.device_info
except db.ReferencePropertyResolveError:
logging.exception('Device deleted for measurement %s',
measurement.key().id())
# Skip this measurement
continue
# catch case where measurement is not supported
if not self.type_to_summary.has_key(measurement.type):
self.type_to_summary[measurement.type] = \
model.ValidationSummary(measurement_type=measurement.type,
error="UnknownType")
# set initial data for validation summary
if not self.type_to_summary[measurement.type].timestamp_start:
self.type_to_summary[measurement.type].timestamp_start = \
util.MicrosecondsSinceEpochToTime(dt_start)
self.type_to_summary[measurement.type].timestamp_end = \
util.MicrosecondsSinceEpochToTime(dt)
self.type_to_summary[measurement.type].record_count = 1
self.type_to_summary[measurement.type].error_count = 0
num_invalid[measurement.type] = 0
else:
self.type_to_summary[measurement.type].record_count += 1
# Make a measurement-specific validation object
try:
validator = MeasurementValidatorFactory.CreateValidator(
measurement)
except RuntimeError:
continue # no validator defined, continue
valid = validator.Validate()
if not valid['valid']:
if not num_invalid.has_key(measurement.type):
num_invalid[measurement.type] = 0
num_invalid[measurement.type] += 1
# print the error details
if self.PRINT_VALUES or (self.PRINT_INVALID
and not valid['valid']):
if not self.validation_results.has_key(
'%s_invalid_detail' % measurement.type):
self.validation_results['%s_invalid_detail' %
measurement.type] = []
self.validation_results[
'%s_invalid_detail' % measurement.type].append(
'Errors: %s <br>\nTime: %s<br>\nDevice: %s<br>\nDetails: %s'
% (", ".join(
valid['error_types']), str(measurement.timestamp),
measurement.device_properties.device_info.id,
validator.PrintData()))
# update error counters
for error in valid['error_types']:
if not error_type_to_count.has_key(measurement.type):
error_type_to_count[measurement.type] = dict()
if not error_type_to_count[measurement.type].has_key(
error):
error_type_to_count[measurement.type][error] = 1
else:
error_type_to_count[measurement.type][error] += 1
# create ValidationEntry for each error
if len(valid['error_types']) > 0:
error_detail = model.ValidationEntry()
#error_detail.summary = self.type_to_summary[measurement.type]
error_detail.measurement = measurement
error_detail.error_types = valid['error_types']
self.type_to_details[measurement.type].append(error_detail)
# update number of invalid measurements
self.validation_results['%s_invalid' % measurement.type] = \
num_invalid[measurement.type]
# update validation summary entity for each data type
for data_type in self.type_to_summary.keys():
if num_invalid.has_key(data_type):
self.type_to_summary[data_type].error_count = num_invalid[
data_type]
if error_type_to_count.has_key(data_type):
self.type_to_summary[data_type].SetErrorByType(
error_type_to_count[data_type])
self.validation_results[data_type + "_count"] = \
self.type_to_summary[data_type].record_count
