def save_statistic(self, bucket, type, name, value, anon, in_average_data=None): args = {'value': value, 'updated': int(time()), 'anon': anon, } # print "starting set_sql_dict" records = yield self.dbconfig.select('statistics', select='*', where=['bucket = ? AND type = ? AND name = ?', bucket, type, name]) if len(records) > 0: # now we need to merge the results. This can be fun. # print "existing stat found: %s" % records[0] if type == 'counter': args['value'] = records[0]['value'] + value results = yield self.dbconfig.update('statistics', args, where=['id = ?', records[0]['id']] ) elif type == 'datapoint': # chance is super rare.... Just replace the value. Probably never happens. results = yield self.dbconfig.update('statistics', args, where=['id = ?', records[0]['id']] ) elif type == 'average': record_average_data = cPickle.loads(str(records[0]['averagedata'])) # print "record_average_data: %s" % record_average_data # print "in_average_data: %s" % in_average_data counts = [record_average_data['count'], in_average_data['count']] medians = [record_average_data['median'], in_average_data['median']] uppers = [record_average_data['upper'], in_average_data['upper']] lowers = [record_average_data['lower'], in_average_data['lower']] upper_90s = [record_average_data['upper_90'], in_average_data['upper_90']] lower_90s = [record_average_data['lower_90'], in_average_data['lower_90']] median_90s = [record_average_data['median_90'], in_average_data['median_90']] # found this weighted averaging method here: # http://stackoverflow.com/questions/29330792/python-weighted-averaging-a-list new_average_data = {} new_average_data['count'] = record_average_data['count'] + in_average_data['count'] new_average_data['median'] = sum(x * y for x, y in zip(medians, counts)) / sum(counts) new_average_data['upper'] = sum(x * y for x, y in zip(uppers, counts)) / sum(counts) new_average_data['lower'] = sum(x * y for x, y in zip(lowers, counts)) / sum(counts) new_average_data['upper_90'] = sum(x * y for x, y in zip(upper_90s, counts)) / sum(counts) new_average_data['lower_90'] = sum(x * y for x, y in zip(lower_90s, counts)) / sum(counts) new_average_data['median_90'] = sum(x * y for x, y in zip(median_90s, counts)) / sum(counts) args['averagedata'] = sqlite3Binary(cPickle.dumps(new_average_data, cPickle.HIGHEST_PROTOCOL)) results = yield self.dbconfig.update('statistics', args, where=['id = ?', records[0]['id']] ) else: pass else: args['bucket'] = bucket args['type'] = type args['name'] = name if type == 'average': args['averagedata'] = sqlite3Binary(cPickle.dumps(in_average_data, cPickle.HIGHEST_PROTOCOL)) # print "saving new SQL record: %s" % args results = yield self.dbconfig.insert('statistics', args, None, 'OR IGNORE' ) returnValue(results)
def save_sql_dict(self, save_all=False): """ Called periodically and on exit to save a dictionary to the SQL database. This allows multiple updates to happen to a dictionary without the overhead of constantly updating the matching SQL record. This can lead to some data loss if data is constantly updating and the system crashes. :param save_all: If true, save all the SQL Dictionaries :return: """ for name, di in self._dictionaries.iteritems(): # logger.warn("save_sql_dict 1") if di['dirty'] or save_all: # logger.warn("save_sql_dict 3 {di}", di=di) safe_data = {} # Sometimes wierd datatype's happen... Not good. for key, item in di['dict'].iteritems(): safe_data[key] = item # logger.warn("save_sql_dict 4 {di}", di=safe_data) save_data = sqlite3Binary(cPickle.dumps(safe_data, cPickle.HIGHEST_PROTOCOL)) yield self._Libraries['localdb'].set_sql_dict(di['component_name'], di['dict_name'], save_data) # print "in save_sql_dict - returned from saving data into sql" self._dictionaries[name]['dirty'] = False if self.unload_defer is not None: self.unload_defer.callback(10)
def process_config(self, inputType, config_item, config_type, msg): logger.debug("in process_config ->> config_item: {config_item}", config_item=config_item) # make sure the command exists if config_item not in self.config_items: logger.warn("ConfigurationUpdate::process_config - '{config_item}' is not a valid configuration item. Skipping.", config_item=config_item) return elif config_item == "gateway_configs": payload = msg['data'] for section in payload: for key in section['Values']: self._Configs.set(section['Section'], key['Key'], key['Value']) elif config_item == "gateway_variable": records = msg['data']["configdata"] sendUpdates = [] for record in records: if self._Libraries['configuration'].get_config_time(record['section'], record['item']) > record['updated']: self._Configs.set(record['section'], record['item'], record['value']) else: #the gateway is newer sendUpdates.append({'section': record['section'], 'item' : record['item'], 'value' : record['value']}) # needs to be implemented on server first!! # if len(sendUpdates): # self.gateway_control.sendQueueAdd(self._generateMessage({'cmd' : 'setGatewayVariables', 'configdata':sendUpdates})) # self._removeFullTableQueue('GatewayVariablesTable') elif config_item in self.config_items: logger.debug("ConfigurationUpdate::process_config - Doing config for: {config_item}", config_item=config_item) configs_db = self.config_items[config_item] data = [] if 'data_type' in msg: if msg['data_type'] == 'object': # a single response logger.debug("Processing single object config response.") data.append(msg['data']) elif msg['data_type'] == 'objects': # An array of responses logger.debug("Processing multiple object config response.") data = msg['data'] else: if isinstance(msg, list): data = msg elif isinstance(msg, dict): data = data.append(msg) else: raise YomboWarning("Cannot process configuration update") tempConfig = {} # Usef for various tracking. Variable depends on config_item being processed. tempIndex = {} # Usef for various tracking. Variable depends on config_item being processed. tempStorage = {} # Usef for various tracking. Variable depends on config_item being processed. save_records = [] for record in data: items = record.items() temp_record = {} for col, val in items: if col not in configs_db['map']: # logger.debug("## Col (%s) not in table.." % col) continue # print "col = %s (%s)" % (col, configs_db['map'][col]) if self._LocalDBLibrary.db_model[configs_db['table']][configs_db['map'][col]]['type'] == "INTEGER": val=int(val) elif self._LocalDBLibrary.db_model[configs_db['table']][configs_db['map'][col]]['type'] == "REAL": val=float(val) elif type(val) is dict: val = sqlite3Binary(cPickle.dumps(val, cPickle.HIGHEST_PROTOCOL)) # temp = (col, decryptPGP(val)) temp_record[configs_db['map'][col]] = val save_records.append(temp_record) # logger.debug("Pre checking nested %s" % config_item) # process any nested items here. print "config_item: %s" % config_item if config_item == 'gateway_modules': if '1' not in tempConfig: tempConfig['1'] = { 'inputType' : 'nested', 'config_item' : 'device_type_modules', } tempConfig['2'] = { 'inputType' : 'nested', 'config_item' : 'command_device_types', } tempConfig['3'] = { 'inputType' : 'nested', 'config_item' : 'device_types', } tempConfig['4'] = { 'inputType' : 'nested', 'config_item' : 'variables', } tempIndex['1'] = [] # device_type_modules tempIndex['2'] = [] # command_device_types tempIndex['3'] = [] # device_types tempIndex['4'] = [] # variables tempStorage['1'] = [] tempStorage['2'] = [] tempStorage['3'] = [] tempStorage['4'] = [] # logger.info("devicetypes: %s" % record['device_types'][) if 'DeviceTypes' in record: # logger.debug("Call nested: {record}" % record=record) for tempDT in record['DeviceTypes']: if tempDT['UUID'] not in tempIndex['3']: # print "adding device type: %s" % tempDT tempIndex['3'].append(tempDT['UUID']) tempStorage['3'].append(tempDT) for dt in record['DeviceTypes']: if dt['UUID'] not in tempIndex['1']: tempStorage['1'].append({ 'device_type_id' : dt['UUID'], 'module_id' : record['UUID'], # record = module 'priority' : dt['Priority'], }) for dtc in dt['Commands']: if dt['UUID'] not in tempIndex['2']: tempStorage['2'].append({ 'device_type_id' : dt['UUID'], #dt = devicetype 'command_id' : dtc['UUID'], #dtc = Commanddevice_types }) # ModuleConfigs if 'ModuleConfigs' in record: for tempGroup in record['ModuleConfigs']: for tempField in tempGroup['Fields']: if tempField['FieldUUID'] not in tempIndex['4']: tempIndex['4'].append(tempField['FieldUUID']) field = { 'VariableType': 'module', 'ForeignUUID' : record['UUID'], # record = module 'VariableUUID' : tempGroup['VariableUUID'], 'Weight' : tempGroup['Weight'], 'DataWeight' : tempField['Weight'], 'MachineLabel' : tempField['MachineLabel'], 'Label' : tempField['Label'], 'Value' : tempField['Value'], 'Updated' : tempField['Updated'], 'Created' : tempField['Created'], } tempStorage['4'].append(field) # end if config_item == 'gateway_modules' elif config_item == 'gateway_devices': print "config record: %s" % record if '1' not in tempConfig: tempConfig['1'] = { 'inputType' : inputType, 'config_item' : 'variables', } tempIndex['1'] = [] # DeviceConfigs tempStorage['1'] = [] # DeviceConfigs if 'DeviceConfigs' in record: for tempGroup in record['DeviceConfigs']: for tempField in tempGroup['Fields']: if tempField['FieldUUID'] not in tempIndex['1']: tempIndex['1'].append(tempField['FieldUUID']) field = { 'VariableType': 'device', 'ForeignUUID' : record['UUID'], # record = device 'VariableUUID' : tempGroup['VariableUUID'], 'Weight' : tempGroup['Weight'], 'DataWeight' : tempField['Weight'], 'MachineLabel' : tempField['MachineLabel'], 'Label' : tempField['Label'], 'Value' : tempField['Value'], 'Updated' : tempField['Updated'], 'Created' : tempField['Created'], } tempStorage['1'].append(field) for key, value in tempStorage.iteritems(): # logger.info("key: {key}, value: {value}", key=key, value=value) self.process_config(tempConfig[key]['inputType'], tempConfig[key]['config_item'], config_type, tempStorage[key]) if len(save_records) > 0: # for record in save_records: # yield self._LocalDBLibrary.insert(configs_db['table'], record) # print "saving records: (%s): %s" % (configs_db['table'], save_records) yield self._LocalDBLibrary.insert_many(configs_db['table'], save_records) else: raise YomboWarning("Unknown type on processing configuration update.") if inputType == "response" and config_type == "full": self._removeFullDownloadQueue("get_" + config_item) self.__incomingConfigQueueCheck()