def ListMeasurements(self, **unused_args): """Handles /measurements REST request.""" # This is very limited and only supports time range and limit # queries. You might want to extend this to filter by other # properties, but for the purpose of measurement archiving # this is all we need. start_time = self.request.get('start_time') end_time = self.request.get('end_time') limit = self.request.get('limit') query = model.Measurement.all() if start_time: dt = util.MicrosecondsSinceEpochToTime(int(start_time)) query.filter('timestamp >=', dt) if end_time: dt = util.MicrosecondsSinceEpochToTime(int(end_time)) query.filter('timestamp <', dt) query.order('timestamp') if limit: results = query.fetch(int(limit)) else: results = query output = util.MeasurementListToDictList(results) self.response.out.write(json.dumps(output))
def CommonExceptions(self, **unused_args): """Returns a list containing the most common 10 exceptions and the number of times they have been reported.""" start_time = self.request.get('start_time') end_time = self.request.get('start_time') limit = self.request.get('limit') entries = model.ValidationEntry.all() logging.log(logging.INFO, "Found %s records" % model.ValidationEntry.all().count(10000)) # TODO(drc): support date queries if start_time: start_time = util.MicrosecondsSinceEpochToTime(int(start_time)) if end_time: end_time = util.MicrosecondsSinceEpochToTime(int(end_time)) if limit: limit = int(limit) else: limit = 1000 error_to_count = dict() for entry in entries.fetch(limit): if entry.measurement.success == False: # Only grab first portion of stack trace if there was an error if not (entry.measurement.GetValue('error') is None): error = "<br>".join( entry.measurement.GetValue('error').split("\n")[0:5]) if not error_to_count.has_key(error): error_to_count[error] = 1 else: error_to_count[error] += 1 sorted_errors = sorted(error_to_count.iteritems(), key=operator.itemgetter(1), reverse=True) return sorted_errors[0:10]
def TimeseriesData(self, **unused_args): """Returns data for the timeseries view in JSON format.""" device_id = self.request.get('device_id') start_time = self.request.get('start_time') end_time = self.request.get('start_time') limit = self.request.get('limit') # Used to trigger permission check unused_device = model.DeviceInfo.GetDeviceWithAcl(device_id) if start_time: start_time = util.MicrosecondsSinceEpochToTime(int(start_time)) if end_time: end_time = util.MicrosecondsSinceEpochToTime(int(end_time)) if limit: limit = int(limit) measurements = model.Measurement.GetMeasurementListWithAcl( limit=limit, device_id=device_id, start_time=start_time, end_time=end_time) tsdata = [] for meas in measurements: ms_time = util.TimeToMicrosecondsSinceEpoch(meas.timestamp) / 1000 rssi = meas.device_properties.rssi or 0 battery = meas.device_properties.battery_level or 0 val = ('new Date(%d)' % ms_time, rssi, battery) tsdata.append(val) self.response.out.write(json.dumps(tsdata))
def ListMeasurements(self, **unused_args): """Handles /measurements REST request.""" # This is very limited and only supports time range and limit # queries. You might want to extend this to filter by other # properties, but for the purpose of measurement archiving # this is all we need. start_time = self.request.get('start_time') end_time = self.request.get('end_time') limit = self.request.get('limit') query = model.Measurement.all() if start_time: dt = util.MicrosecondsSinceEpochToTime(int(start_time)) query.filter('timestamp >=', dt) if end_time: dt = util.MicrosecondsSinceEpochToTime(int(end_time)) query.filter('timestamp <', dt) query.order('timestamp') if limit: results = query.fetch(int(limit)) else: results = query output = [] for measurement in results: # Need to catch case where device has been deleted try: unused_device_info = measurement.device_properties.device_info except db.ReferencePropertyResolveError: logging.exception('Device deleted for measurement %s', measurement.key().id()) # Skip this measurement continue # Need to catch case where task has been deleted try: unused_task = measurement.task except db.ReferencePropertyResolveError: measurement.task = None measurement.put() mdict = util.ConvertToDict(measurement, timestamps_in_microseconds=True) # Fill in additional fields mdict['id'] = str(measurement.key().id()) mdict['parameters'] = measurement.Params() mdict['values'] = measurement.Values() if 'task' in mdict and mdict['task'] is not None: mdict['task']['id'] = str(measurement.GetTaskID()) mdict['task']['parameters'] = measurement.task.Params() output.append(mdict) self.response.out.write(json.dumps(output))
def _Archive(self, **unused_args): """Processes parameters and packages the data into a file-like object. This method does the bulk of the heavy lifting. It makes sense of the request parameters, serializes the data, generates a name used to encapsulate the result, and makes the call to compress the file-like object. URL Args: device_id: A string key for a device in the datastore. start_time: The timestamp for the earliest measurement (microseconds UTC) end_time: The timestamp for the latest measurement (microseconds UTC) All of the URL Args are optional and in the case where none are given no restrictions are assumed and all the data is returned. Since these handlers should never be called by users it should be understood that this case should be avoided. Returns: A name that is suitable for describing the contents of the file based on the parameters for the request, and a file-like object containing the measurement data. Raises: No exceptions handled here. No new exceptions generated here. """ #TODO(mdw) Unit test needed. # Make sense of parameters device_id = self.request.get('device_id') start_time = self.request.get('start_time') end_time = self.request.get('end_time') if start_time: start = util.MicrosecondsSinceEpochToTime(int(start_time)) else: start = None if end_time: end = util.MicrosecondsSinceEpochToTime(int(end_time)) else: end = None # Get data based on parameters model_list = GetMeasurementDictList(device_id, start, end) # Serialize the data data = {'Measurement': json.dumps(model_list)} # Generate directory/file name based on parameters archive_dir = ParametersToFileNameBase(device_id, start_time, end_time) # For some reason there was a problem with Unicode chars in the request archive_dir = archive_dir.encode('ascii', 'ignore') #NOTE: This is a multiple return. return archive_dir, archive_util.ArchiveCompress(data, directory=archive_dir)
def TimeseriesData(self, **unused_args): """Returns data for the timeseries view in JSON format.""" start_time = self.request.get('start_time') end_time = self.request.get('start_time') limit = self.request.get('limit') sample_type = self.request.get('type') summaries = model.ValidationSummary.all() if start_time: start_time = util.MicrosecondsSinceEpochToTime(int(start_time)) summaries.filter('timestamp_start > ', start_time) if end_time: end_time = util.MicrosecondsSinceEpochToTime(int(end_time)) summaries.filter('timestamp_end < ', end_time) if limit: limit = int(limit) else: limit = 1000 summaries.filter('timestamp_start > ', 0) summaries.order('timestamp_start') time_to_type_to_count = dict() tsdata = [] # group data by timestamp for timeline printing for summary in summaries.fetch(limit): ms_time = util.TimeToMicrosecondsSinceEpoch( summary.timestamp_start) / 1e3 if not time_to_type_to_count.has_key(ms_time): time_to_type_to_count[ms_time] = dict() if sample_type: time_to_type_to_count[ms_time][summary.measurement_type] = \ getattr(summary, sample_type) else: time_to_type_to_count[ms_time][summary.measurement_type] = \ summary.record_count # gather data into timeline-friendly output format for time, type_to_count in time_to_type_to_count.items(): row = ['new Date(%d)' % time] for meas, name in measurement.MEASUREMENT_TYPES: if type_to_count.has_key(meas): row.append(type_to_count[meas]) else: row.append(0) tsdata.append(row) self.response.out.write(json.dumps(tsdata))
def JSON_DECODE_timestamp(self, inputval): try: self.timestamp = util.MicrosecondsSinceEpochToTime(int(inputval)) except ValueError: logging.exception('Error occurs while converting timestamp ' '%s to integer', inputval) self.timestamp = None
def DashboardDetail(self, **unused_args): """Returns a dict of time to measurement type to count.""" start_time = self.request.get('start_time') end_time = self.request.get('start_time') limit = self.request.get('limit') entries = model.ValidationEntry.all() if start_time: start_time = util.MicrosecondsSinceEpochToTime(int(start_time)) if end_time: end_time = util.MicrosecondsSinceEpochToTime(int(end_time)) if limit: limit = int(limit) else: limit = 1000 # TODO(drc): Incorporate date limits. time_to_type_to_cnt = SortedDict() # group by time for ent in entries.fetch(limit): ms_time = ent.summary.timestamp_start meas_type = ent.summary.measurement_type time_to_type_to_cnt.setdefault(ms_time, dict()).setdefault( meas_type, {'details': []}) time_to_type_to_cnt[ms_time][meas_type][ 'count'] = ent.summary.error_count # links to ids for eventually showing more detail time_to_type_to_cnt[ms_time][meas_type]['details'].append([ ent.measurement.key().id(), ent.measurement.device_properties.device_info.id ]) # now sort by time sorted_results = SortedDict() for k in sorted(time_to_type_to_cnt.iterkeys()): sorted_results[k] = time_to_type_to_cnt[k] return sorted_results
def Validate(self, **unused_args): """Main handler for the validation view. Note that this method is called from a request that is restricted to admin privileges. Args (HTTP request parameters): start_time: start time for validation period (ISO8601 format) end_time: end time for validation period (ISO8601 format) iters: (optional) Number of days to do bulk validation on. (int) use_webpage: (optional) Show validation results in HTML. If set to false, send an e-mail with results. (boolean) worker: (optional) True if running as a queued task. (boolean) Returns: Result of validation, or nothing. Raises: No exceptions handled here. No new exceptions generated here. """ iters = self.request.get('iters') start_time = self.request.get('start_time') end_time = self.request.get('end_time') # if iterating, set up and enqueue validation tasks while iters and int(iters) > 1: start_time = 24 * 60 * 60 * 1000 * 1000 + \ util.TimeToMicrosecondsSinceEpoch(util.StringToTime(start_time)) start_time = util.MicrosecondsSinceEpochToTime(start_time) start_time = util.TimeToString(start_time) end_time = 24 * 60 * 60 * 1000 * 1000 + \ util.TimeToMicrosecondsSinceEpoch(util.StringToTime(end_time)) end_time = util.MicrosecondsSinceEpochToTime(end_time) end_time = util.TimeToString(end_time) # Add the task to the 'validation' queue. taskqueue.add( url='/validation/data?worker=true&start_time=%s&end_time=%s' % (start_time, end_time), method='GET', queue_name='validation') iters = int(iters) - 1 # return here if iterating using task queue if iters: self.response.out.write("{Success:true}") return # contains validation results for printing self.validation_results = dict() # support only the measurements specified in MEASUREMENT_TYPES for mtype, name in measurement.MEASUREMENT_TYPES: self.type_to_summary[mtype] = \ model.ValidationSummary(measurement_type=mtype) self.type_to_details[mtype] = [] # validate all the data in one pass self._DoValidation() # validation results are in type_to_details, now write them to datastore for mtype, data in self.type_to_summary.items(): data.put( ) # must put summary before putting details that reference them if self.type_to_details.has_key(mtype): for detail in self.type_to_details[mtype]: detail.summary = data detail.put() # if this was a queued task, return here if self.request.get('worker'): self.response.out.write("{Success:true}") return # for display purposes, render HTML of results html = template.render('templates/validation.html', self.validation_results) # send to response, or e-mail user if self.request.get('use_webpage'): self.response.out.write(html) else: message = mail.EmailMessage(sender=config.VALIDATION_EMAIL_SENDER, subject="Daily validation results") message.to = config.VALIDATION_EMAIL_RECIPIENT message.body = html message.html = html message.send()
def _DoValidation(self): """Gets all the records for a specified type, subject to request parameters. Args: None. This reads from the measurement request Returns: Nothing. It writes results to an instance variable. Raises: No exceptions handled here. No new exceptions generated here. """ start_time = self.request.get('start_time') end_time = self.request.get('end_time') limit = self.request.get('limit') # manually specified parameters for testing if self.TESTING: limit = 100 start_time = util.TimeToMicrosecondsSinceEpoch( util.StringToTime("2012-03-20T00:00:00Z")) end_time = util.TimeToMicrosecondsSinceEpoch( util.StringToTime("2012-03-21T00:00:00Z")) # set up query filters according to parameters query = model.Measurement.all() if start_time: dt_start = util.TimeToMicrosecondsSinceEpoch( util.StringToTime(start_time)) else: dt_start = util.TimeToMicrosecondsSinceEpoch( datetime.datetime.utcfromtimestamp(time.time()) - datetime.timedelta(days=1)) self.validation_results['start_time'] = \ util.MicrosecondsSinceEpochToTime(dt_start) query.filter('timestamp >=', dt_start) if end_time: dt = util.TimeToMicrosecondsSinceEpoch(util.StringToTime(end_time)) else: dt = util.TimeToMicrosecondsSinceEpoch( datetime.datetime.utcfromtimestamp(time.time())) self.validation_results[ 'end_time'] = util.MicrosecondsSinceEpochToTime(dt) query.filter('timestamp <', dt) query.order('timestamp') if limit: results = query.fetch(int(limit)) else: results = query error_type_to_count = dict() # map of error type to count num_invalid = dict() # map of measurement type to count of invalid for measurement in results: # Need to catch case where device has been deleted try: unused_device_info = measurement.device_properties.device_info except db.ReferencePropertyResolveError: logging.exception('Device deleted for measurement %s', measurement.key().id()) # Skip this measurement continue # catch case where measurement is not supported if not self.type_to_summary.has_key(measurement.type): self.type_to_summary[measurement.type] = \ model.ValidationSummary(measurement_type=measurement.type, error="UnknownType") # set initial data for validation summary if not self.type_to_summary[measurement.type].timestamp_start: self.type_to_summary[measurement.type].timestamp_start = \ util.MicrosecondsSinceEpochToTime(dt_start) self.type_to_summary[measurement.type].timestamp_end = \ util.MicrosecondsSinceEpochToTime(dt) self.type_to_summary[measurement.type].record_count = 1 self.type_to_summary[measurement.type].error_count = 0 num_invalid[measurement.type] = 0 else: self.type_to_summary[measurement.type].record_count += 1 # Make a measurement-specific validation object try: validator = MeasurementValidatorFactory.CreateValidator( measurement) except RuntimeError: continue # no validator defined, continue valid = validator.Validate() if not valid['valid']: if not num_invalid.has_key(measurement.type): num_invalid[measurement.type] = 0 num_invalid[measurement.type] += 1 # print the error details if self.PRINT_VALUES or (self.PRINT_INVALID and not valid['valid']): if not self.validation_results.has_key( '%s_invalid_detail' % measurement.type): self.validation_results['%s_invalid_detail' % measurement.type] = [] self.validation_results[ '%s_invalid_detail' % measurement.type].append( 'Errors: %s <br>\nTime: %s<br>\nDevice: %s<br>\nDetails: %s' % (", ".join( valid['error_types']), str(measurement.timestamp), measurement.device_properties.device_info.id, validator.PrintData())) # update error counters for error in valid['error_types']: if not error_type_to_count.has_key(measurement.type): error_type_to_count[measurement.type] = dict() if not error_type_to_count[measurement.type].has_key( error): error_type_to_count[measurement.type][error] = 1 else: error_type_to_count[measurement.type][error] += 1 # create ValidationEntry for each error if len(valid['error_types']) > 0: error_detail = model.ValidationEntry() #error_detail.summary = self.type_to_summary[measurement.type] error_detail.measurement = measurement error_detail.error_types = valid['error_types'] self.type_to_details[measurement.type].append(error_detail) # update number of invalid measurements self.validation_results['%s_invalid' % measurement.type] = \ num_invalid[measurement.type] # update validation summary entity for each data type for data_type in self.type_to_summary.keys(): if num_invalid.has_key(data_type): self.type_to_summary[data_type].error_count = num_invalid[ data_type] if error_type_to_count.has_key(data_type): self.type_to_summary[data_type].SetErrorByType( error_type_to_count[data_type]) self.validation_results[data_type + "_count"] = \ self.type_to_summary[data_type].record_count