naeDict['ebu3b'] = ['505', '506']
deviceList = naeDict[buildingName]
beginTime = datetime(2016, 1, 18)
endTime = datetime(2016, 1, 25)
# Init raw set
missingDataList = list()
for nae in deviceList:
devices = sensors_dict[nae]
for sensor in devices['objs'][1:]:
h_obj = sensor['props']
srcid = nae + '_' + str(h_obj['type']) + '_' + str(h_obj['instance'])
print srcid
try:
uuid = bdm.get_sensor_uuids(
context={'source_identifier': srcid})[0]
ts = bdm.get_sensor_ts(uuid, 'PresentValue', beginTime, endTime)
resampledTs = resample_data(ts, beginTime, endTime, 'nextval')
filename = rawdataDir + srcid + '.shelve'
# with open(filename, 'wb') as fp:
# pickle.dump(resampledTs, fp)
writer = shelve.open(filename)
writer['data'] = resampledTs
except:
missingDataList.append(srcid)
print missingDataList
pd.Series(missingDataList).to_csv('missingdatalist.csv')
naeDict = dict()
naeDict['ebu3b'] = ['505', '506']
deviceList = naeDict[buildingName]
beginTime = datetime(2016,1,18)
endTime = datetime(2016,1,25)
# Init raw set
missingDataList = list()
for nae in deviceList:
devices = sensors_dict[nae]
for sensor in devices['objs'][1:]:
h_obj = sensor['props']
srcid = nae + '_' + str(h_obj['type']) + '_' + str(h_obj['instance'])
print srcid
try:
uuid = bdm.get_sensor_uuids(context={'source_identifier':srcid})[0]
ts = bdm.get_sensor_ts(uuid, 'PresentValue', beginTime, endTime)
resampledTs = resample_data(ts, beginTime, endTime, 'nextval')
filename = rawdataDir + srcid + '.shelve'
# with open(filename, 'wb') as fp:
# pickle.dump(resampledTs, fp)
writer = shelve.open(filename)
writer['data'] = resampledTs
except:
missingDataList.append(srcid)
print missingDataList
pd.Series(missingDataList).to_csv('missingdatalist.csv')
class Actuator(object):
__metaclass__ = ABCMeta
minLatency = None # 0 minimum
inputType = None # should be selected among above type classes
#lowerActuators = list()
#higherActuators = list()
affectingDependencyDict = dict(
) # values of this dict are each actuator's minLatency
affectedDependencyDict = dict(
) # values of this dict are this actuator's minLatency
controlFlag = False
uuid = None
name = None
bdm = None
sensorType = None # ==actuator
resetVal = None
depMapFile = 'metadata/dependency_map.json'
def __init__(self, minLatency):
# minLatency(datetime) ->
self.minLatency = minLatency
self.bdm = BDWrapper()
# depMap = pickle.load(open(self.depMapFile, 'rb'))
with open(self.depMapFile, 'rb') as fp:
depMap = json.load(fp)
if self.uuid in depMap.keys():
for depUuid in depMap[self.uuid]:
self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
self.affectedDependencyDict[depUuid] = timedelta(minutes=10)
@abstractmethod
def set_value(self, val, tp):
self.controlFlag = True
if self.inputType.validate(val):
self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
else:
print "Failed to validate a value of " + self.zone + '\'s Common Setpoint to ' + str(
val)
@abstractmethod
def get_value(self, beginTime, endTime):
return self.bdm.get_sensor_ts(self.uuid, self.sensorType, beginTime,
endTime)
@abstractmethod #Should this be abm?
def reset_value(self, val, tp):
self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
self.controlFlag = False
def get_latest_value(self, now):
result = self.bdm.get_sensor_ts(self.uuid, self.sensorType,
now - timedelta(hours=6),
now + timedelta(minutes=10))
if result.empty:
return None
else:
return result.tail(1).values[0], result.tail(1).index[0]
def get_second_latest_value(self, now):
result = self.bdm.get_sensor_ts(self.uuid, self.sensorType,
now - timedelta(hours=6),
now + timedelta(minutes=10))
if len(result) <= 1:
return None
else:
return result.tail(2).values[1], result.tail(2).index[1]
def check_control_flag(self):
return self.controlFlag
def check_dependency(self, commDict, setTime):
if (commDict['set_time'] >
setTime - self.minLatency) and commDict['uuid'] == self.uuid:
return True
else:
return False
def get_dependency(self, uuid):
if uuid in self.affectingDependencyDict.keys():
return self.affectingDependencyDict[uuid]
elif uuid in self.affectedDependencyDict.keys():
return self.affectedDependencyDict[uuid]
else:
return None
def get_dependent_actu_list(self):
return self.affectingDependencyDict.keys(
) + self.affectedDependencyDict.keys()
def get_longest_dependency(self):
return max(max(self.affectedDependencyDict.values()),
max(self.affectingDependencyDict.values()))
def validate_input(self, given):
return self.inputType.validate(given)
class Analyzer:
bdm = None
expLogColl = None
#timeGran = timedelta(minutes=5)
timeGran = timedelta(minutes=2)
actuNames = None
sensorNames = None
zonelist = None
feater = None
clust = None
def __init__(self):
self.actuNames = ActuatorNames()
self.sensorNames = SensorNames()
self.bdm = BDWrapper()
self.expLogColl = CollectionWrapper('experience_log')
#self.zonelist = self.csv2list('metadata/partialzonelist.csv')
self.zonelist = self.csv2list('metadata/zonelist.csv')
self.feater = FeatureExtractor()
self.clust = Clusterer()
def csv2list(self, filename):
outputList = list()
with open(filename, 'r') as fp:
reader = csv.reader(fp, delimiter=',')
for row in reader:
outputList.append(row[0])
return outputList
def get_actuator_uuid(self, zone=None, actuType=None):
context = dict()
if zone != None:
context['room']=zone
if actuType != None:
context['template']=actuType
uuids = self.bdm.get_sensor_uuids(context)
if len(uuids)>1:
raise QRError('Many uuids are found', context)
elif len(uuids)==0:
raise QRError('No uuid is found', context)
else:
return uuids[0]
def normalize_data_avg(self, rawData, beginTime, endTime):
procData = pd.Series({beginTime:float(rawData[0])})
tp = beginTime
while tp<=endTime:
tp = tp+self.timeGran
leftSeries = rawData[:tp]
if len(leftSeries)>0:
idx = len(leftSeries)-1
leftVal = leftSeries[idx]
leftIdx = leftSeries.index[idx]
else:
leftVal = None
rightSeries = rawData[tp:]
if len(rightSeries)>0:
rightVal = rightSeries[0]
rightIdx = rightSeries.index[0]
else:
rightVal = None
if rightVal==None and leftVal!=None:
newVal = leftVal
elif rightVal!=None and leftVal==None:
newVal = rightVal
elif tp==leftIdx:
newVal = leftVal
elif tp==rightIdx:
newVal = rightVal
elif rightVal!=None and leftVal!=None:
leftDist = (tp - leftIdx).total_seconds()
rightDist = (rightIdx - tp).total_seconds()
newVal = (leftVal*rightDist+rightVal*leftDist)/(rightDist+leftDist)
else:
print "ERROR: no data found in raw data"
newVal = None
newData = pd.Series({tp:newVal})
procData = procData.append(newData)
return procData
def normalize_data_nextval_deprecated(self, rawData, beginTime, endTime):
procData = pd.Series({beginTime:float(rawData[0])})
tp = beginTime
while tp<=endTime:
tp = tp+self.timeGran
leftSeries = rawData[:tp]
if len(leftSeries)>0:
idx = len(leftSeries)-1
leftVal = leftSeries[idx]
leftIdx = leftSeries.index[idx]
else:
leftVal = None
rightSeries = rawData[tp:]
if len(rightSeries)>0:
rightVal = rightSeries[0]
rightIdx = rightSeries.index[0]
else:
rightVal = None
if rightVal != None:
newVal = rightVal
else:
newVal = leftVal
newData = pd.Series({tp:newVal})
procData = procData.append(newData)
return procData
def normalize_data(self, rawData, beginTime, endTime, normType):
rawData = rawData[beginTime:endTime]
if not beginTime in rawData.index:
rawData[beginTime] = rawData.head(1)[0]
rawData = rawData.sort_index()
if not endTime in rawData.index:
rawData[endTime] = rawData.tail(1)[0]
rawData = rawData.sort_index()
if normType=='nextval':
procData = rawData.resample('2Min', fill_method='pad')
elif normType=='avg':
procData = rawData.resample('2Min', how='mean')
else:
procData = None
return procData
def receive_a_sensor(self, zone, actuType, beginTime, endTime, normType):
print zone, actuType
uuid = self.get_actuator_uuid(zone, actuType)
rawData = self.bdm.get_sensor_ts(uuid, 'PresentValue', beginTime, endTime)
if actuType!=self.actuNames.damperCommand:
rawData = self.remove_negativeone(rawData)
procData = self.normalize_data(rawData, beginTime, endTime, normType)
return procData
def receive_entire_sensors_notstore(self, beginTime, endTime, normType, exceptZoneList=[]):
#TODO: Should be parallelized here
dataDict = dict()
for zone in self.zonelist:
if not zone in exceptZoneList:
dataDict[zone] = self.receive_zone_sensors(zone, beginTime, endTime, normType)
return dataDict
def receive_entire_sensors(self, beginTime, endTime, filename, normType, exceptZoneList=[]):
# filename='data/'+beginTime.isoformat()[0:-7].replace(':','_') + '.pkl'
dataDict = self.receive_entire_sensors_notstore(beginTime, endTime, normType, exceptZoneList=exceptZoneList)
with open(filename, 'wb') as fp:
pickle.dump(dataDict, fp)
# json.dump(dataDict,fp)
def clustering(self, inputData, dataDict):
fftFeat = self.feater.get_fft_features(inputData, dataDict)
minmaxFeat = self.feater.get_minmax_features(dataDict)
dtwFeat = self.feater.get_dtw_features(inputData, dataDict)
freqFeat = self.feater.get_freq_features(inputData, dataDict)
featDict = dict()
for zone in self.zonelist:
featList = list()
featList.append(fftFeat[zone])
featList.append(minmaxFeat[zone])
featList.append(dtwFeat[zone])
#featList.append(freqFeat[zone])
featDict[zone] = featList
print featDict['RM-4132']
return self.clust.cluster_kmeans(featDict)
def remove_negativeone(self, data):
if -1 in data.values:
indices = np.where(data==-1)
for idx in indices:
data[idx] = data[idx-1]
return data
def receive_zone_sensors(self, zone, beginTime, endTime, normType):
zoneDict = dict()
for actuType in self.actuNames.nameList+self.sensorNames.nameList:
if actuType=='Actual Supply Flow':
pass
try:
uuid = self.get_actuator_uuid(zone, actuType)
except QRError:
continue
# if actuType == self.actuNames.commonSetpoint:
# wcad = self.receive_a_sensor(zone, 'Warm Cool Adjust', beginTime, endTime, normType)
# data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
# data = data + wcad
# pass
if actuType != self.actuNames.damperCommand:
if actuType==self.actuNames.occupiedCommand:
pass
data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
else:
data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
zoneDict[actuType] = data
return zoneDict
def store_zone_sensors(self, zone, beginTime, endTime, normType, filename):
data = self.receive_zone_sensors(zone, beginTime, endTime, normType)
# with open(filename, 'wb') as fp:
# w = csv.DictWriter(fp, data.keys())
# w.writeheader()
# w.writerow(data)
for key, val in data.iteritems():
val.to_csv('rm4132.csv', header=key, mode='a')
def store_minmax_dict(self):
minDict = defaultdict(dict)
maxDict = defaultdict(dict)
beginTime = datetime(2015,2,1)
endTime = datetime(2015,9,1)
shortBeginTime = datetime(2015,8,1)
shortEndTime = datetime(2015,8,2)
for zone in self.zonelist:
for pointType in self.actuNames.nameList+self.sensorNames.nameList:
try:
if pointType=='Occupied Command':
minDict[zone][pointType] = 1
maxDict[zone][pointType] = 3
elif pointType=='Cooling Command':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Cooling Command' or pointType=='Heating Command':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Occupied Clg Min' or pointType=='Occupied Htg Flow' or pointType=='Cooling Max Flow':
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
minDict[zone][pointType] = min(data)
maxDict[zone][pointType] = max(data)
elif pointType=='Temp Occ Sts':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 1
elif pointType=='Reheat Valve Command':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Actual Supply Flow' or pointType=='Actual Sup Flow SP':
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
maxFlow = data[0]
minDict[zone][pointType] = 0
maxDict[zone][pointType] = maxFlow
elif pointType=='Damper Position':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Damper Command':
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
meanData = np.mean(data)
stdData = np.std(data)
meanAgain = np.mean(data[np.logical_and(data<=meanData+2*stdData, data>=meanData-2*stdData)])
minDict[zone][pointType] = meanData-2*stdData
maxDict[zone][pointType] = meanData+2*stdData
else:
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', beginTime, endTime)
minDict[zone][pointType] = min(data)
maxDict[zone][pointType] = max(data)
except:
print "Something is wrong"
pass
with open('metadata/mindict.pkl', 'wb') as fp:
pickle.dump(minDict, fp)
with open('metadata/maxdict.pkl', 'wb') as fp:
pickle.dump(maxDict, fp)
class Actuator(object): __metaclass__ = ABCMeta minLatency = None # 0 minimum inputType = None # should be selected among above type classes #lowerActuators = list() #higherActuators = list() affectingDependencyDict = dict() # values of this dict are each actuator's minLatency affectedDependencyDict = dict() # values of this dict are this actuator's minLatency controlFlag = False uuid = None name = None bdm = None sensorType = None # ==actuator resetVal = None depMapFile = 'metadata/dependency_map.json' def __init__(self, minLatency): # minLatency(datetime) -> self.minLatency = minLatency self.bdm = BDWrapper() # depMap = pickle.load(open(self.depMapFile, 'rb')) with open(self.depMapFile,'rb') as fp: depMap = json.load(fp) if self.uuid in depMap.keys(): for depUuid in depMap[self.uuid]: self.affectingDependencyDict[depUuid] = timedelta(minutes=10) self.affectedDependencyDict[depUuid] = timedelta(minutes=10) @abstractmethod def set_value(self, val, tp): self.controlFlag = True if self.inputType.validate(val): self.bdm.set_sensor(self.uuid, self.sensorType, tp, val) else: print "Failed to validate a value of " + self.zone + '\'s Common Setpoint to ' + str(val) @abstractmethod def get_value(self, beginTime, endTime): return self.bdm.get_sensor_ts(self.uuid, self.sensorType, beginTime, endTime) @abstractmethod #Should this be abm? def reset_value(self, val, tp): self.bdm.set_sensor(self.uuid, self.sensorType, tp, val) self.controlFlag = False def get_latest_value(self, now): result = self.bdm.get_sensor_ts(self.uuid, self.sensorType, now-timedelta(hours=6), now+timedelta(minutes=10)) if result.empty: return None else: return result.tail(1).values[0], result.tail(1).index[0] def get_second_latest_value(self,now): result = self.bdm.get_sensor_ts(self.uuid, self.sensorType, now-timedelta(hours=6), now+timedelta(minutes=10)) if len(result)<=1: return None else: return result.tail(2).values[1], result.tail(2).index[1] def check_control_flag(self): return self.controlFlag def check_dependency(self, commDict, setTime): if (commDict['set_time']> setTime-self.minLatency) and commDict['uuid']==self.uuid: return True else: return False def get_dependency(self, uuid): if uuid in self.affectingDependencyDict.keys(): return self.affectingDependencyDict[uuid] elif uuid in self.affectedDependencyDict.keys(): return self.affectedDependencyDict[uuid] else: return None def get_dependent_actu_list(self): return self.affectingDependencyDict.keys() + self.affectedDependencyDict.keys() def get_longest_dependency(self): return max(max(self.affectedDependencyDict.values()),max(self.affectingDependencyDict.values())) def validate_input(self, given): return self.inputType.validate(given)
