def save_statistic(self, bucket, type, name, value, anon, in_average_data=None):
args = {'value': value,
'updated': int(time()),
'anon': anon,
}
# print "starting set_sql_dict"
records = yield self.dbconfig.select('statistics', select='*', where=['bucket = ? AND type = ? AND name = ?', bucket, type, name])
if len(records) > 0: # now we need to merge the results. This can be fun.
# print "existing stat found: %s" % records[0]
if type == 'counter':
args['value'] = records[0]['value'] + value
results = yield self.dbconfig.update('statistics', args, where=['id = ?', records[0]['id']] )
elif type == 'datapoint': # chance is super rare.... Just replace the value. Probably never happens.
results = yield self.dbconfig.update('statistics', args, where=['id = ?', records[0]['id']] )
elif type == 'average':
record_average_data = cPickle.loads(str(records[0]['averagedata']))
# print "record_average_data: %s" % record_average_data
# print "in_average_data: %s" % in_average_data
counts = [record_average_data['count'], in_average_data['count']]
medians = [record_average_data['median'], in_average_data['median']]
uppers = [record_average_data['upper'], in_average_data['upper']]
lowers = [record_average_data['lower'], in_average_data['lower']]
upper_90s = [record_average_data['upper_90'], in_average_data['upper_90']]
lower_90s = [record_average_data['lower_90'], in_average_data['lower_90']]
median_90s = [record_average_data['median_90'], in_average_data['median_90']]
# found this weighted averaging method here:
# http://stackoverflow.com/questions/29330792/python-weighted-averaging-a-list
new_average_data = {}
new_average_data['count'] = record_average_data['count'] + in_average_data['count']
new_average_data['median'] = sum(x * y for x, y in zip(medians, counts)) / sum(counts)
new_average_data['upper'] = sum(x * y for x, y in zip(uppers, counts)) / sum(counts)
new_average_data['lower'] = sum(x * y for x, y in zip(lowers, counts)) / sum(counts)
new_average_data['upper_90'] = sum(x * y for x, y in zip(upper_90s, counts)) / sum(counts)
new_average_data['lower_90'] = sum(x * y for x, y in zip(lower_90s, counts)) / sum(counts)
new_average_data['median_90'] = sum(x * y for x, y in zip(median_90s, counts)) / sum(counts)
args['averagedata'] = sqlite3Binary(cPickle.dumps(new_average_data, cPickle.HIGHEST_PROTOCOL))
results = yield self.dbconfig.update('statistics', args, where=['id = ?', records[0]['id']] )
else:
pass
else:
args['bucket'] = bucket
args['type'] = type
args['name'] = name
if type == 'average':
args['averagedata'] = sqlite3Binary(cPickle.dumps(in_average_data, cPickle.HIGHEST_PROTOCOL))
# print "saving new SQL record: %s" % args
results = yield self.dbconfig.insert('statistics', args, None, 'OR IGNORE' )
returnValue(results)
def save_sql_dict(self, save_all=False):
"""
Called periodically and on exit to save a dictionary to the SQL database.
This allows multiple updates to happen to a dictionary without the overhead of constantly updating the
matching SQL record. This can lead to some data loss if data is constantly updating and the system crashes.
:param save_all: If true, save all the SQL Dictionaries
:return:
"""
for name, di in self._dictionaries.iteritems():
# logger.warn("save_sql_dict 1")
if di['dirty'] or save_all:
# logger.warn("save_sql_dict 3 {di}", di=di)
safe_data = {} # Sometimes wierd datatype's happen... Not good.
for key, item in di['dict'].iteritems():
safe_data[key] = item
# logger.warn("save_sql_dict 4 {di}", di=safe_data)
save_data = sqlite3Binary(cPickle.dumps(safe_data, cPickle.HIGHEST_PROTOCOL))
yield self._Libraries['localdb'].set_sql_dict(di['component_name'],
di['dict_name'], save_data)
# print "in save_sql_dict - returned from saving data into sql"
self._dictionaries[name]['dirty'] = False
if self.unload_defer is not None:
self.unload_defer.callback(10)
def process_config(self, inputType, config_item, config_type, msg):
logger.debug("in process_config ->> config_item: {config_item}", config_item=config_item)
# make sure the command exists
if config_item not in self.config_items:
logger.warn("ConfigurationUpdate::process_config - '{config_item}' is not a valid configuration item. Skipping.", config_item=config_item)
return
elif config_item == "gateway_configs":
payload = msg['data']
for section in payload:
for key in section['Values']:
self._Configs.set(section['Section'], key['Key'], key['Value'])
elif config_item == "gateway_variable":
records = msg['data']["configdata"]
sendUpdates = []
for record in records:
if self._Libraries['configuration'].get_config_time(record['section'], record['item']) > record['updated']:
self._Configs.set(record['section'], record['item'], record['value'])
else: #the gateway is newer
sendUpdates.append({'section': record['section'],
'item' : record['item'],
'value' : record['value']})
# needs to be implemented on server first!!
# if len(sendUpdates):
# self.gateway_control.sendQueueAdd(self._generateMessage({'cmd' : 'setGatewayVariables', 'configdata':sendUpdates}))
# self._removeFullTableQueue('GatewayVariablesTable')
elif config_item in self.config_items:
logger.debug("ConfigurationUpdate::process_config - Doing config for: {config_item}", config_item=config_item)
configs_db = self.config_items[config_item]
data = []
if 'data_type' in msg:
if msg['data_type'] == 'object': # a single response
logger.debug("Processing single object config response.")
data.append(msg['data'])
elif msg['data_type'] == 'objects': # An array of responses
logger.debug("Processing multiple object config response.")
data = msg['data']
else:
if isinstance(msg, list):
data = msg
elif isinstance(msg, dict):
data = data.append(msg)
else:
raise YomboWarning("Cannot process configuration update")
tempConfig = {} # Usef for various tracking. Variable depends on config_item being processed.
tempIndex = {} # Usef for various tracking. Variable depends on config_item being processed.
tempStorage = {} # Usef for various tracking. Variable depends on config_item being processed.
save_records = []
for record in data:
items = record.items()
temp_record = {}
for col, val in items:
if col not in configs_db['map']:
# logger.debug("## Col (%s) not in table.." % col)
continue
# print "col = %s (%s)" % (col, configs_db['map'][col])
if self._LocalDBLibrary.db_model[configs_db['table']][configs_db['map'][col]]['type'] == "INTEGER":
val=int(val)
elif self._LocalDBLibrary.db_model[configs_db['table']][configs_db['map'][col]]['type'] == "REAL":
val=float(val)
elif type(val) is dict:
val = sqlite3Binary(cPickle.dumps(val, cPickle.HIGHEST_PROTOCOL))
# temp = (col, decryptPGP(val))
temp_record[configs_db['map'][col]] = val
save_records.append(temp_record)
# logger.debug("Pre checking nested %s" % config_item)
# process any nested items here.
print "config_item: %s" % config_item
if config_item == 'gateway_modules':
if '1' not in tempConfig:
tempConfig['1'] = {
'inputType' : 'nested',
'config_item' : 'device_type_modules',
}
tempConfig['2'] = {
'inputType' : 'nested',
'config_item' : 'command_device_types',
}
tempConfig['3'] = {
'inputType' : 'nested',
'config_item' : 'device_types',
}
tempConfig['4'] = {
'inputType' : 'nested',
'config_item' : 'variables',
}
tempIndex['1'] = [] # device_type_modules
tempIndex['2'] = [] # command_device_types
tempIndex['3'] = [] # device_types
tempIndex['4'] = [] # variables
tempStorage['1'] = []
tempStorage['2'] = []
tempStorage['3'] = []
tempStorage['4'] = []
# logger.info("devicetypes: %s" % record['device_types'][)
if 'DeviceTypes' in record:
# logger.debug("Call nested: {record}" % record=record)
for tempDT in record['DeviceTypes']:
if tempDT['UUID'] not in tempIndex['3']:
# print "adding device type: %s" % tempDT
tempIndex['3'].append(tempDT['UUID'])
tempStorage['3'].append(tempDT)
for dt in record['DeviceTypes']:
if dt['UUID'] not in tempIndex['1']:
tempStorage['1'].append({
'device_type_id' : dt['UUID'],
'module_id' : record['UUID'], # record = module
'priority' : dt['Priority'],
})
for dtc in dt['Commands']:
if dt['UUID'] not in tempIndex['2']:
tempStorage['2'].append({
'device_type_id' : dt['UUID'], #dt = devicetype
'command_id' : dtc['UUID'], #dtc = Commanddevice_types
})
# ModuleConfigs
if 'ModuleConfigs' in record:
for tempGroup in record['ModuleConfigs']:
for tempField in tempGroup['Fields']:
if tempField['FieldUUID'] not in tempIndex['4']:
tempIndex['4'].append(tempField['FieldUUID'])
field = {
'VariableType': 'module',
'ForeignUUID' : record['UUID'], # record = module
'VariableUUID' : tempGroup['VariableUUID'],
'Weight' : tempGroup['Weight'],
'DataWeight' : tempField['Weight'],
'MachineLabel' : tempField['MachineLabel'],
'Label' : tempField['Label'],
'Value' : tempField['Value'],
'Updated' : tempField['Updated'],
'Created' : tempField['Created'],
}
tempStorage['4'].append(field)
# end if config_item == 'gateway_modules'
elif config_item == 'gateway_devices':
print "config record: %s" % record
if '1' not in tempConfig:
tempConfig['1'] = {
'inputType' : inputType,
'config_item' : 'variables',
}
tempIndex['1'] = [] # DeviceConfigs
tempStorage['1'] = []
# DeviceConfigs
if 'DeviceConfigs' in record:
for tempGroup in record['DeviceConfigs']:
for tempField in tempGroup['Fields']:
if tempField['FieldUUID'] not in tempIndex['1']:
tempIndex['1'].append(tempField['FieldUUID'])
field = {
'VariableType': 'device',
'ForeignUUID' : record['UUID'], # record = device
'VariableUUID' : tempGroup['VariableUUID'],
'Weight' : tempGroup['Weight'],
'DataWeight' : tempField['Weight'],
'MachineLabel' : tempField['MachineLabel'],
'Label' : tempField['Label'],
'Value' : tempField['Value'],
'Updated' : tempField['Updated'],
'Created' : tempField['Created'],
}
tempStorage['1'].append(field)
for key, value in tempStorage.iteritems():
# logger.info("key: {key}, value: {value}", key=key, value=value)
self.process_config(tempConfig[key]['inputType'], tempConfig[key]['config_item'], config_type, tempStorage[key])
if len(save_records) > 0:
# for record in save_records:
# yield self._LocalDBLibrary.insert(configs_db['table'], record)
# print "saving records: (%s): %s" % (configs_db['table'], save_records)
yield self._LocalDBLibrary.insert_many(configs_db['table'], save_records)
else:
raise YomboWarning("Unknown type on processing configuration update.")
if inputType == "response" and config_type == "full":
self._removeFullDownloadQueue("get_" + config_item)
self.__incomingConfigQueueCheck()
