def __init__(self, minLatency):
# minLatency(datetime) ->
self.minLatency = minLatency
self.bdm = BDWrapper()
# depMap = pickle.load(open(self.depMapFile, 'rb'))
with open(self.depMapFile, 'rb') as fp:
depMap = json.load(fp)
if self.uuid in depMap.keys():
for depUuid in depMap[self.uuid]:
self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
self.affectedDependencyDict[depUuid] = timedelta(minutes=10)
class CommonSetpoint(Actuator):
bdm = BDWrapper()
zone = None
template = None
#TODO: Is it okay to set minVal and maxVal arbitrarily outside this class?
def __init__ (self, name, uuid, minVal, maxVal, zone):
self.name = name
self.uuid = uuid
super(CommonSetpoint, self).__init__(timedelta(minutes=10))
self.inputType = master_actuator.TempType(minVal, maxVal)
self.zone = zone
self.template = actuNames.commonSetpoint
self.sensorType = 'PresentValue'
def set_value(self, val, tp):
if val=='ZT':
ztuuid = self.bdm.get_sensor_uuids({'room':self.zone, 'template':sensorNames.zoneTemperature})[0]
now = datetime.now()
currZT = self.bdm.get_sensor_ts(ztuuid, 'PresentValue', now-timedelta(hours=1), now).tail().tolist()[0]
val = currZT
else:
pass
super(CommonSetpoint, self).set_value(val, tp+timedelta(minutes=1))
def get_value(self, beginTime, endTime):
return super(CommonSetpoint, self).get_value(beginTime, endTime)
def reset_value(self, val, tp):
super(CommonSetpoint, self).reset_value(val, tp)
#TODO: Do I need this?
def set_reset_value(self, resetVal=-1):
self.resetVal = resetVal
def __init__(self):
self.actuNames = ActuatorNames()
self.sensorNames = SensorNames()
self.bdm = BDWrapper()
self.expLogColl = CollectionWrapper('experience_log')
#self.zonelist = self.csv2list('metadata/partialzonelist.csv')
self.zonelist = self.csv2list('metadata/zonelist.csv')
self.feater = FeatureExtractor()
self.clust = Clusterer()
def __init__(self, minLatency): # minLatency(datetime) -> self.minLatency = minLatency self.bdm = BDWrapper() # depMap = pickle.load(open(self.depMapFile, 'rb')) with open(self.depMapFile,'rb') as fp: depMap = json.load(fp) if self.uuid in depMap.keys(): for depUuid in depMap[self.uuid]: self.affectingDependencyDict[depUuid] = timedelta(minutes=10) self.affectedDependencyDict[depUuid] = timedelta(minutes=10)
def __init__(self):
self.ztTrackZoneList = list()
self.ntpClient = ntplib.NTPClient()
self.statColl = CollectionWrapper('status')
self.expLogColl = CollectionWrapper('experiment_log')
self.ntpActivateTime = self.dummyBeginTime
logging.basicConfig(filename='log/debug'+datetime.now().isoformat()[0:-7].replace(':','_') + '.log',level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s')
#logger = logging.getLogger()
logging.debug('Quiver initialization')
self.bdm = BDWrapper()
self.update_time_offset()
self.zonelist = basic.csv2list('metadata/zonelist.csv')
self.depMapFile = 'metadata/dependency_map.json'
# requests.packages.urllib3.disable_warnings()
# Create pid file for monitoring
pid = str(os.getpid())
class Quiver:
ntpURL = 'ntp.ucsd.edu'
timeOffset = timedelta(0)
ntpClient = None
actuDict= dict()
actuNames = ActuatorNames()
futureCommColl = None # This is a collection for future command sequence. If some of the commands are issued, they are removed from here.
expLogColl = None # This is a collection for log of control. If a command is issued, it is added to here with relevant information.
statColl = None # This is a collection for rollback. If a command is issued, its corresponding rollback command is added here.
relinquishVal = -1
ambulanceConn = None
ntpActivateTime = None
dummyBeginTime = datetime(2000,1,1)
dummyEndTime = datetime(2030,12,31,0,0,0)
bdm = None
ackLatency = timedelta(minutes=10)
statusExpiration = timedelta(hours=24)
zonelist = None
logger = None
ztTrackZoneList = None
def __init__(self):
self.ztTrackZoneList = list()
self.ntpClient = ntplib.NTPClient()
self.statColl = CollectionWrapper('status')
self.expLogColl = CollectionWrapper('experiment_log')
self.ntpActivateTime = self.dummyBeginTime
logging.basicConfig(filename='log/debug'+datetime.now().isoformat()[0:-7].replace(':','_') + '.log',level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s')
#logger = logging.getLogger()
logging.debug('Quiver initialization')
self.bdm = BDWrapper()
self.update_time_offset()
self.zonelist = basic.csv2list('metadata/zonelist.csv')
self.depMapFile = 'metadata/dependency_map.json'
# requests.packages.urllib3.disable_warnings()
# Create pid file for monitoring
pid = str(os.getpid())
#pidfile = "C:\\temp\\quiver.pid"
# if os.path.isfile(pidfile):
# print "%s already exists, exiting" %pidfile
# sys.exit()
# else:
# file(pidfile, 'w').write(pid)
#TODO: Above should be enabled in final version
def __del__(self):
pass
# Construct dependency map for all points
# Run this when you run this for the fisrt time
# TODO: For now, it finds all the points in the corresponding zone.
def init_dependency_map(self):
depZoneMap = defaultdict(list)
depUuidMap = defaultdict(list)
for zone in self.zonelist:
for actuType in self.actuNames.nameList:
try:
uuid = self.get_actuator_uuid(zone, actuType)
depZoneMap[zone].append(uuid)
except QRError as e:
print "There is no sensor corresponding to ", zone, actuType
for zone, depList in depZoneMap.iteritems():
for uuid in depList:
for depUuid in depList:
if uuid != depUuid:
depUuidMap[uuid].append(depUuid)
with open(self.depMapFile,'w') as fp:
json.dump(depUuidMap, fp)
def notify_systemfault(self):
content = "Quiver control system bas been down at " + self.now().isoformat()
self.notify_email(content)
def notify_email(self, content):
server = smtplib.SMTP(emailauth.smtpURL)
msg = MIMEText('"'+content+'"')
msg['Subject']='Alert: Quiver is down'
msg['From'] = emailauth.fromaddr
msg['To'] = ",".join(emailauth.toaddrs)
server.starttls()
server.login(emailauth.username, emailauth.password)
server.sendmail(emailauth.fromaddr, emailauth.toaddrs, msg.as_string())
server.quit()
# This is for NTP with UCSD BMS.
# TODO: It should be done in OS itself. Any suggestion?
def update_time_offset(self):
ntpRequest = self.ntpClient.request(self.ntpURL)
# ntpRequest.tx_time
ntpTime = datetime.strptime(time.ctime(ntpRequest.tx_time), "%a %b %d %H:%M:%S %Y")
self.timeOffset = ntpTime - datetime.now()
return ntpTime
# Currently, zone and actuator type is the easiest information for applications to use, so I use it to get uuid, but can be generalized.
def get_actuator_uuid(self, zone=None, actuType=None):
context = dict()
if zone != None:
context['room']=zone
if actuType != None:
context['template']=actuType
uuids = self.bdm.get_sensor_uuids(context)
if len(uuids)>1:
raise QRError('Many uuids are found', context)
elif len(uuids)==0:
raise QRError('No uuid is found', context)
else:
return uuids[0]
def get_actuator_name(self,zone=None,actuType=None):
context = dict()
if zone != None:
context['room']=zone
if actuType != None:
context['template']=actuType
uuids = self.bdm.get_sensor_names(context)
if len(uuids)>1:
raise QRError('Many uuids are found', context)
elif len(uuids)==0:
raise QRError('No uuid is found', context)
else:
return uuids[0]
def load_reset_seq(self, endTime):
query = {'reset_time':{'$lte':endTime}}
futureSeq = self.statColl.pop_dataframe(query)
return futureSeq
def actuator_exist(self, uuid):
# zone(string), actuatortype(string) -> existing?(boolean)
if uuid in self.actuDict.keys():
return True
else:
return False
def rollback_to_original_setting(self):
resetSeq = self.load_reset_seq(self.dummyEndTime)
for row in resetSeq.iterrows():
currTime = self.now()
zone = row[1]['zone']
actuType = row[1]['actuator_type']
resetVal = row[1]['reset_value']
uuid = row[1]['uuid']
actuator = self.actuDict[uuid]
actuator.reset_value(resetVal, currTime)
self.statColl.remove_all()
def validate_command_seq_freq(self,seq):
# seq(pd.DataFrame) -> valid?(boolean)
#TODO: This does not consider reset commands. However, it should be considered later
baseInvalidMsg = "Test sequence is invalid because "
for row in seq.iterrows():
zone = row[1]['zone']
actuType = row[1]['actuator_type']
uuid = row[1]['uuid']
actuator = self.actuDict[uuid]
minLatency = actuator.minLatency
setTime = row[1]['set_time']
inrangeRowsIdx = np.bitwise_and(seq['set_time']<setTime+minLatency, seq['set_time']>setTime-minLatency)
inrangeRowsIdx = np.bitwise_and(inrangeRowsIdx, seq['uuid']==uuid)
inrangeRowsIdx[row[0]] = False
inrangeRows = seq[inrangeRowsIdx.values.tolist()]
resetRows = self.statColl.load_dataframe({'uuid':uuid})
loggedRows = self.expLogColl.load_dataframe({'$and':[{'set_time':{'$gte':setTime-actuator.minLatency}},{'uuid':uuid}]})
inrangeRows = pd.concat([inrangeRows, resetRows, loggedRows])
for inrangeRow in inrangeRows.iterrows():
if inrangeRow[1]['zone']==zone and inrangeRow[1]['actuator_type']==actuType:
print baseInvalidMsg + str(row[1]) + ' is overlapped with ' + str(inrangeRow[1])
return row[1]
return pd.DataFrame({})
def validate_command_seq_dependency(self, seq, minExpLatency):
# seq(pd.DataFrame) -> valid?(boolean)
#baseInvalidMsg = "Test sequence is invalid because "
for row in seq.iterrows():
zone = row[1]['zone']
actuType = row[1]['actuator_type']
uuid = row[1]['uuid']
actuator = self.actuDict[uuid]
minLatency = actuator.minLatency
setTime = row[1]['set_time']
inrangeRowsIdx = np.bitwise_and(seq['set_time']<setTime, seq['set_time']>setTime-minExpLatency)
inrangeRows = seq[inrangeRowsIdx.values.tolist()]
resetRows = self.statColl.load_dataframe({})
loggedRows = self.expLogColl.load_dataframe({'set_time':{'$gte':setTime-minExpLatency}})
inrangeRows = pd.concat([inrangeRows, resetRows, loggedRows])
for inrangeRow in inrangeRows.iterrows():
if inrangeRow[1]['zone']==zone and actuator.get_dependency(inrangeRow[1]['actuator_type'])!=None:
# print baseInvalidMsg + str(row[1]) + ' is dependent on ' + str(inrangeRow[1])
return row[1]
return pd.DataFrame({})
def static_validate(self, seq):
invalidFreqCommand = self.validate_command_seq_freq(seq)
if not invalidFreqCommand.empty:
return invalidFreqCommand
invalidDepCommand = self.validate_command_seq_dependency(seq, timedelta(minutes=5)) #TODO: This minExpLatency should be set to 1 hour later
if not invalidDepCommand.empty:
return invalidDepCommand
return pd.DataFrame({})
def validate_batch(self, seq):
# Validate each command
for seqRow in seq.iterrows():
zone = seqRow[1]['zone']
actuType = seqRow[1]['actuator_type']
setVal = seqRow[1]['set_value']
uuid = self.get_actuator_uuid(zone, actuType)
name= self.get_actuator_name(zone, actuType)
seqRow[1]['uuid'] = uuid
seqRow[1]['name'] = name
if not uuid in self.actuDict.keys():
actuator = metaactuators.make_actuator(uuid, name, zone, actuType)
if actuator == None:
logging.error("Incorrect actuator type: %s", actuType)
raise QRError('Incorrect actuator type', actuType)
self.actuDict[uuid] = actuator
actuator = self.actuDict[uuid]
# Validation 1: Check input range
if not actuator.validate_input(setVal):
logging.error("Input value is not in correct range: %s", repr(seqRow[1]))
raise QRError("Input value is not in correct range", seqRow[1])
# Validation 2: Check if there is a dependent command in given batch
for otherSeqRow in seq.iterrows():
if seqRow[0]!=otherSeqRow[0]:
if actuator.check_dependency(otherSeqRow[1], self.now()):
logging.error("A command is dependent on a command in the given sequence: %s", repr(seqRow[1]) + repr(otherSeqRow[1]))
raise QRError("A command is dependent on a command in the given sequence", repr(seqRow[1]) + repr(otherSeqRow[1]))
elif otherSeqRow[1]['uuid'] == uuid:
logging.error('A command has same target equipment with another: %s', repr(seqRow[1])+ repr(otherSeqRow[1]))
raise QRError('A command has same target equipment with another', repr(seqRow[1])+ repr(otherSeqRow[1]))
# Validation 3: Check if there is a dependent command in current status
queryDep = {'set_time':{'$gte':self.now()-actuator.minLatency}}
depCommands = self.statColl.load_dataframe(queryDep)
for commRow in depCommands.iterrows():
if actuator.check_dependency(commRow[1], self.now()):
logging.error("A command is dependent on current status: %s", repr(seqRow[1])+repr(commRow[1]))
raise QRError("A command is dependent on current status", repr(seqRow[1])+repr(commRow[1]))
seq.loc[seqRow[0]] = seqRow[1]
def now(self):
currTime = datetime.now()
currTime = currTime + self.timeOffset
return currTime
# PRIMARY function.
def issue_seq(self, seq):
logging.debug('Start issuing: \n%s', repr(seq))
self.validate_batch(seq)
logging.debug('Validation is done')
#TODO: Check if updated seq is returned
for row in seq.iterrows():
idx = row[0]
zone = row[1]['zone']
setVal = row[1]['set_value']
actuType = row[1]['actuator_type']
uuid = row[1]['uuid']
name = row[1]['name']
actuator = self.actuDict[uuid]
now = self.now()
origVal = actuator.get_latest_value(now)[0]
seq.loc[idx, 'original_value'] = origVal
if actuator.check_control_flag():
query = {'uuid':uuid}
resetVal = self.statColl.load_dataframe(query).tail(1)
#TODO: This should become more safe. e.g., what if that is no data in statColl?
resetVal = float(resetVal['reset_value'])
else:
resetVal = origVal
seq.loc[idx, 'reset_value'] = resetVal
setTime = self.now()
seq.loc[idx, 'set_time'] = setTime
if setVal == -1:
if actuType in [self.actuNames.commonSetpoint]:
setVal = float(self.statColl.load_dataframe({'uuid':uuid}).tail(1)['reset_value'][0])
actuator.reset_value(setVal, setTime)
logging.debug('Reset a value: %s by %s', actuType, str(setVal))
else:
actuator.set_value(setVal, setTime) #TODO: This should not work in test stage
logging.debug('Set a value: %s by %s', actuType, str(setVal))
self.ack_issue(seq)
logging.debug("Logging done")
#TODO: initialIssueFlagList should be checked if is implemented correctly
def ack_issue(self, seq):
if len(seq)==0:
return None
seq.index = range(0,len(seq)) # Just to make it sure (need to remove?)
issueFlagList = np.array([False]*len(seq))
initialIssueFlagList = np.array([True]*len(seq))
uploadedTimeList = list()
resendInterval = timedelta(minutes=6)
maxWaitTime = resendInterval * 2
# Init uploadedTimeList
for row in seq.iterrows():
uuid = row[1]['uuid']
actuator = self.actuDict[uuid]
latestVal, setTime = actuator.get_latest_value(self.now())
setVal = row[1]['set_value']
# TODO: Think about timing here
# What if a value is reset then temporal current value is restored?
# Temporarily do not check the case where setVal==-1.
if setVal=='-1':
pass
elif (setVal!=-1 and latestVal != setVal):
latestVal, setTime = actuator.get_second_latest_value(self.now())
if setVal!=-1 and latestVal != setVal:
initialIssueFlagList[row[0]] = False
logging.error("A command is not issued initially: \n%s", repr(row[1]))
#raise QRError('Initial upload to BD is failed', row[1])
uploadedTimeList.append(setTime)
uploadedTimeList = np.array(uploadedTimeList)
# Receive ack
maxWaitDatetime = max(uploadedTimeList)+maxWaitTime
ackInterval = 30 # secounds
while maxWaitDatetime>=self.now():
for row in seq.iterrows():
idx = row[0]
if initialIssueFlagList[idx]==False:
continue
if issueFlagList[idx]==True:
continue
uuid = row[1]['uuid']
setVal = row[1]['set_value']
origVal = row[1]['original_value']
actuator = self.actuDict[uuid]
if setVal == -1:
ackVal = row[1]['reset_value']
else:
ackVal = row[1]['set_value']
actuType = row[1]['actuator_type']
actuator = self.actuDict[uuid]
currT = self.now()
currVal, newSetTime = actuator.get_latest_value(self.now())
logging.debug("ack: uploadTime: %s, downloadTime: %s", str(uploadedTimeList[idx]), str(newSetTime))
if (setVal!=-1 and currVal==ackVal and newSetTime!=uploadedTimeList[idx]) or (setVal==-1 and currVal!=-1):
issueFlagList[idx] = True
seq.loc[idx, 'set_time'] = uploadedTimeList[idx]
logging.debug("Received a ack of a command: \n%s", repr(row[1]))
if setVal==-1:
#actuator.reset_value(currVal, uploadedTimeList[idx])
pass
continue
now = self.now()
if now>=uploadedTimeList[idx]+resendInterval:
actuator.set_value(origVal, uploadedTimeList[idx])
setTime = self.now()
if setVal==-1:
if actuType in [self.actuNames.commonSetpoint]:
setVal = row[1]['reset_value']
actuator.reset_value(setVal,setTime)
else:
actuator.set_value(setVal, setTime)
logging.debug("Resend a command due to timeout: \n%s", repr(row[1]))
uploadedTimeList[idx] = setTime
if not (False in issueFlagList):
break
time.sleep(ackInterval)
for row in seq.iterrows():
if issueFlagList[row[0]] == True:
self.reflect_an_issue_to_db(row[1])
# for idx, flag in enumerate(issueFlagList):
# if not flag:
# raise QRError('Some commands are unable to be uploaded', seq[idx])
if False in issueFlagList or False in initialIssueFlagList:
logging.error('Some commands are unable to be uploaded: %s', repr(seq[np.logical_not(issueFlagList)]+seq[np.logical_not(initialIssueFlagList)]))
raise QRError('Some commands are unable to be uploaded', seq[np.logical_not(issueFlagList)]+seq[np.logical_not(initialIssueFlagList)])
def reflect_an_issue_to_db(self, commDict):
zone = commDict['zone']
setVal = commDict['set_value']
actuType = commDict['actuator_type']
uuid = commDict['uuid']
name = commDict['name']
resetVal = commDict['reset_value']
setTime = commDict['set_time']
origVal = commDict['original_value']
actuator = self.actuDict[uuid]
self.statColl.pop_dataframe({'uuid':uuid})
statusRow = StatusRow(uuid, name, setTime=setTime, setVal=setVal, resetVal=resetVal, actuType=actuType, underControl=actuator.check_control_flag())
self.statColl.store_row(statusRow)
expLogRow = ExpLogRow(uuid, name, setTime=setTime, setVal=setVal, origVal=origVal)
self.expLogColl.store_row(expLogRow)
def reset_seq(self, seq):
for row in seq.iterrows():
#TODO: validate_in_log
print row
resetVal = row[1]['reset_value']
actuType = row[1]['actuator_type']
uuid = row[1]['uuid']
name = row[1]['name']
actuator = self.actuDict[uuid]
now = self.now()
origVal = actuator.get_value(now-timedelta(hours=1), now).tail(1)[0]
actuator.reset_value(resetVal, resetTime)
expLogRow = ExpLogRow(uuid, name, setTime=None, setVal=None, resetVal=now, origVal=origVal)
print expLogRow
self.expLogColl.store_row(expLogRow)
if not self.issue_ack(seq, False).empty:
raise QRError('A reset command cannot be set at BACNet', seq)
def top_ntp(self):
ntpLatency = timedelta(minutes=30)
if self.ntpActivateTime<=self.now():
self.update_time_offset()
self.ntpActivateTime = self.now() + ntpLatency
def system_close_common_behavior(self):
pass
def system_refresh(self):
self.expLogColl.remove_all()
self.statColl.remove_all()
# rollback does not rollback to original state but rollback by "-1"
def emergent_rollback(self):
queryAll = {"under_control":True}
resetQueue = self.statColl.load_dataframe(queryAll)
resetQueue = resetQueue.sort(columns='set_time', axis='index')
maxConcurrentResetNum = 10
resetInterval = 10*60 # 10 minutes
if len(resetQueue)==0:
return None
resetSeq= defaultdict(dict)
#resetSeq = [[]]
# Make actuators to reset and get earliest set time dependent on reset_queue
earliestDepTime = self.dummyEndTime
for row in resetQueue.iterrows():
uuid = row[1]['uuid']
name = row[1]['name']
actuType = row[1]['actuator_type']
if not uuid in self.actuDict.keys():
actuator = metaactuators.make_actuator(uuid,name,actuType=actuType)
if actuator == None:
#raise QRError('Incorrect actuator type', actuType)
print("Incorrect actuator type: ", actuType)
continue
self.actuDict[uuid] = actuator
setTime = row[1]['set_time']
dependentTime = setTime - actuator.get_longest_dependency()
if earliestDepTime > dependentTime:
earliestDepTime = dependentTime
# Construct Reset Sequence
# Filter redundant controls and align with dependency
for row in resetQueue.iterrows():
uuid = row[1]['uuid']
inQFlag = False
for l in resetSeq.values():
if uuid in l.keys():
inQFlag = True
if inQFlag == True:
continue
actuator = self.actuDict[uuid]
lastKey = 0
now = self.now()
row[1]['original_value'] = actuator.get_value(now-timedelta(hours=1), now).tail(1)[0]
insertedFlag = False
for k,l in resetSeq.iteritems():
depList = actuator.get_dependent_actu_list()
if not bool(set(depList) & set(l.keys())) and len(l)<maxConcurrentResetNum:
resetSeq[k][uuid] = row[1].to_dict()
insertedFlag = True
lastKey = k
lastKey += 1
if not insertedFlag:
resetSeq[lastKey][uuid] = row[1].to_dict()
for key, stats in resetSeq.iteritems():
oneResetSeq = pd.DataFrame()
for uuid, stat in stats.iteritems():
oneResetSeq = pd.concat([oneResetSeq, pd.DataFrame(stat, index=[0])])
resetSeq[key] = oneResetSeq
#TODO: need to add checking acknowledgement
# Rollback
for k, l in resetSeq.iteritems():
for key, stat in l.iterrows():
# stat = row[1]
uuid = stat['uuid']
name = stat['name']
actuType = stat['actuator_type']
if actuType in [self.actuNames.commonSetpoint]:
setVal = stat['reset_value']
else:
setVal = -1
actuator = self.actuDict[uuid]
now = self.now()
origVal = stat['original_value']
setTime = self.now()
expLogRow = ExpLogRow(uuid, name, setTime=setTime, setVal=setVal, origVal=origVal)
actuator.reset_value(setVal, setTime)
self.ack_issue(l)
time.sleep(resetInterval)
def get_currest_status(self):
return self.statColl.load_dataframe({'under_control':True})
def output_exp_log(self):
self.expLogColl.to_csv()
def add_zone_zt_tacking(self,zone):
self.ztTrackZoneList.append(zone)
def remove_zone_zt_tacking(self,zone):
try:
self.ztTrackZoneList.remove(zone)
except QRError as e:
raise QRError('Failed to remove zone from ztTrackZonelist: ', zone)
def resample_data(rawData, beginTime, endTime, sampleMethod):
rawData = rawData[beginTime:endTime]
if not beginTime in rawData.index:
rawData[beginTime] = rawData.head(1)[0]
rawData = rawData.sort_index()
if not endTime in rawData.index:
rawData[endTime] = rawData.tail(1)[0]
rawData = rawData.sort_index()
if sampleMethod == 'nextval':
procData = rawData.resample('2Min', how='pad')
return procData
bdm = BDWrapper()
sensors_dict = shelve.open('metadata/bacnet_devices.shelve', 'r')
#buildingName = 'ebu3b'
buildingName = sys.argv[1]
rawdataDir = 'rawdata/' + buildingName + '/'
naeDict = dict()
naeDict['ebu3b'] = ['505', '506']
deviceList = naeDict[buildingName]
beginTime = datetime(2016, 1, 18)
endTime = datetime(2016, 1, 25)
# Init raw set
from bd_wrapper import BDWrapper
def resample_data(rawData, beginTime, endTime, sampleMethod):
rawData = rawData[beginTime:endTime]
if not beginTime in rawData.index:
rawData[beginTime] = rawData.head(1)[0]
rawData = rawData.sort_index()
if not endTime in rawData.index:
rawData[endTime] = rawData.tail(1)[0]
rawData = rawData.sort_index()
if sampleMethod== 'nextval':
procData = rawData.resample('2Min', how='pad')
return procData
bdm = BDWrapper()
sensors_dict = shelve.open('metadata/bacnet_devices.shelve','r')
#buildingName = 'ebu3b'
buildingName = sys.argv[1]
rawdataDir = 'rawdata/'+buildingName+'/'
naeDict = dict()
naeDict['ebu3b'] = ['505', '506']
deviceList = naeDict[buildingName]
beginTime = datetime(2016,1,18)
endTime = datetime(2016,1,25)
class Analyzer:
bdm = None
expLogColl = None
#timeGran = timedelta(minutes=5)
timeGran = timedelta(minutes=2)
actuNames = None
sensorNames = None
zonelist = None
feater = None
clust = None
def __init__(self):
self.actuNames = ActuatorNames()
self.sensorNames = SensorNames()
self.bdm = BDWrapper()
self.expLogColl = CollectionWrapper('experience_log')
#self.zonelist = self.csv2list('metadata/partialzonelist.csv')
self.zonelist = self.csv2list('metadata/zonelist.csv')
self.feater = FeatureExtractor()
self.clust = Clusterer()
def csv2list(self, filename):
outputList = list()
with open(filename, 'r') as fp:
reader = csv.reader(fp, delimiter=',')
for row in reader:
outputList.append(row[0])
return outputList
def get_actuator_uuid(self, zone=None, actuType=None):
context = dict()
if zone != None:
context['room']=zone
if actuType != None:
context['template']=actuType
uuids = self.bdm.get_sensor_uuids(context)
if len(uuids)>1:
raise QRError('Many uuids are found', context)
elif len(uuids)==0:
raise QRError('No uuid is found', context)
else:
return uuids[0]
def normalize_data_avg(self, rawData, beginTime, endTime):
procData = pd.Series({beginTime:float(rawData[0])})
tp = beginTime
while tp<=endTime:
tp = tp+self.timeGran
leftSeries = rawData[:tp]
if len(leftSeries)>0:
idx = len(leftSeries)-1
leftVal = leftSeries[idx]
leftIdx = leftSeries.index[idx]
else:
leftVal = None
rightSeries = rawData[tp:]
if len(rightSeries)>0:
rightVal = rightSeries[0]
rightIdx = rightSeries.index[0]
else:
rightVal = None
if rightVal==None and leftVal!=None:
newVal = leftVal
elif rightVal!=None and leftVal==None:
newVal = rightVal
elif tp==leftIdx:
newVal = leftVal
elif tp==rightIdx:
newVal = rightVal
elif rightVal!=None and leftVal!=None:
leftDist = (tp - leftIdx).total_seconds()
rightDist = (rightIdx - tp).total_seconds()
newVal = (leftVal*rightDist+rightVal*leftDist)/(rightDist+leftDist)
else:
print "ERROR: no data found in raw data"
newVal = None
newData = pd.Series({tp:newVal})
procData = procData.append(newData)
return procData
def normalize_data_nextval_deprecated(self, rawData, beginTime, endTime):
procData = pd.Series({beginTime:float(rawData[0])})
tp = beginTime
while tp<=endTime:
tp = tp+self.timeGran
leftSeries = rawData[:tp]
if len(leftSeries)>0:
idx = len(leftSeries)-1
leftVal = leftSeries[idx]
leftIdx = leftSeries.index[idx]
else:
leftVal = None
rightSeries = rawData[tp:]
if len(rightSeries)>0:
rightVal = rightSeries[0]
rightIdx = rightSeries.index[0]
else:
rightVal = None
if rightVal != None:
newVal = rightVal
else:
newVal = leftVal
newData = pd.Series({tp:newVal})
procData = procData.append(newData)
return procData
def normalize_data(self, rawData, beginTime, endTime, normType):
rawData = rawData[beginTime:endTime]
if not beginTime in rawData.index:
rawData[beginTime] = rawData.head(1)[0]
rawData = rawData.sort_index()
if not endTime in rawData.index:
rawData[endTime] = rawData.tail(1)[0]
rawData = rawData.sort_index()
if normType=='nextval':
procData = rawData.resample('2Min', fill_method='pad')
elif normType=='avg':
procData = rawData.resample('2Min', how='mean')
else:
procData = None
return procData
def receive_a_sensor(self, zone, actuType, beginTime, endTime, normType):
print zone, actuType
uuid = self.get_actuator_uuid(zone, actuType)
rawData = self.bdm.get_sensor_ts(uuid, 'PresentValue', beginTime, endTime)
if actuType!=self.actuNames.damperCommand:
rawData = self.remove_negativeone(rawData)
procData = self.normalize_data(rawData, beginTime, endTime, normType)
return procData
def receive_entire_sensors_notstore(self, beginTime, endTime, normType, exceptZoneList=[]):
#TODO: Should be parallelized here
dataDict = dict()
for zone in self.zonelist:
if not zone in exceptZoneList:
dataDict[zone] = self.receive_zone_sensors(zone, beginTime, endTime, normType)
return dataDict
def receive_entire_sensors(self, beginTime, endTime, filename, normType, exceptZoneList=[]):
# filename='data/'+beginTime.isoformat()[0:-7].replace(':','_') + '.pkl'
dataDict = self.receive_entire_sensors_notstore(beginTime, endTime, normType, exceptZoneList=exceptZoneList)
with open(filename, 'wb') as fp:
pickle.dump(dataDict, fp)
# json.dump(dataDict,fp)
def clustering(self, inputData, dataDict):
fftFeat = self.feater.get_fft_features(inputData, dataDict)
minmaxFeat = self.feater.get_minmax_features(dataDict)
dtwFeat = self.feater.get_dtw_features(inputData, dataDict)
freqFeat = self.feater.get_freq_features(inputData, dataDict)
featDict = dict()
for zone in self.zonelist:
featList = list()
featList.append(fftFeat[zone])
featList.append(minmaxFeat[zone])
featList.append(dtwFeat[zone])
#featList.append(freqFeat[zone])
featDict[zone] = featList
print featDict['RM-4132']
return self.clust.cluster_kmeans(featDict)
def remove_negativeone(self, data):
if -1 in data.values:
indices = np.where(data==-1)
for idx in indices:
data[idx] = data[idx-1]
return data
def receive_zone_sensors(self, zone, beginTime, endTime, normType):
zoneDict = dict()
for actuType in self.actuNames.nameList+self.sensorNames.nameList:
if actuType=='Actual Supply Flow':
pass
try:
uuid = self.get_actuator_uuid(zone, actuType)
except QRError:
continue
# if actuType == self.actuNames.commonSetpoint:
# wcad = self.receive_a_sensor(zone, 'Warm Cool Adjust', beginTime, endTime, normType)
# data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
# data = data + wcad
# pass
if actuType != self.actuNames.damperCommand:
if actuType==self.actuNames.occupiedCommand:
pass
data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
else:
data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
zoneDict[actuType] = data
return zoneDict
def store_zone_sensors(self, zone, beginTime, endTime, normType, filename):
data = self.receive_zone_sensors(zone, beginTime, endTime, normType)
# with open(filename, 'wb') as fp:
# w = csv.DictWriter(fp, data.keys())
# w.writeheader()
# w.writerow(data)
for key, val in data.iteritems():
val.to_csv('rm4132.csv', header=key, mode='a')
def store_minmax_dict(self):
minDict = defaultdict(dict)
maxDict = defaultdict(dict)
beginTime = datetime(2015,2,1)
endTime = datetime(2015,9,1)
shortBeginTime = datetime(2015,8,1)
shortEndTime = datetime(2015,8,2)
for zone in self.zonelist:
for pointType in self.actuNames.nameList+self.sensorNames.nameList:
try:
if pointType=='Occupied Command':
minDict[zone][pointType] = 1
maxDict[zone][pointType] = 3
elif pointType=='Cooling Command':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Cooling Command' or pointType=='Heating Command':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Occupied Clg Min' or pointType=='Occupied Htg Flow' or pointType=='Cooling Max Flow':
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
minDict[zone][pointType] = min(data)
maxDict[zone][pointType] = max(data)
elif pointType=='Temp Occ Sts':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 1
elif pointType=='Reheat Valve Command':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Actual Supply Flow' or pointType=='Actual Sup Flow SP':
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
maxFlow = data[0]
minDict[zone][pointType] = 0
maxDict[zone][pointType] = maxFlow
elif pointType=='Damper Position':
minDict[zone][pointType] = 0
maxDict[zone][pointType] = 100
elif pointType=='Damper Command':
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
meanData = np.mean(data)
stdData = np.std(data)
meanAgain = np.mean(data[np.logical_and(data<=meanData+2*stdData, data>=meanData-2*stdData)])
minDict[zone][pointType] = meanData-2*stdData
maxDict[zone][pointType] = meanData+2*stdData
else:
uuid = self.get_actuator_uuid(zone, pointType)
data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', beginTime, endTime)
minDict[zone][pointType] = min(data)
maxDict[zone][pointType] = max(data)
except:
print "Something is wrong"
pass
with open('metadata/mindict.pkl', 'wb') as fp:
pickle.dump(minDict, fp)
with open('metadata/maxdict.pkl', 'wb') as fp:
pickle.dump(maxDict, fp)
class Actuator(object):
__metaclass__ = ABCMeta
minLatency = None # 0 minimum
inputType = None # should be selected among above type classes
#lowerActuators = list()
#higherActuators = list()
affectingDependencyDict = dict(
) # values of this dict are each actuator's minLatency
affectedDependencyDict = dict(
) # values of this dict are this actuator's minLatency
controlFlag = False
uuid = None
name = None
bdm = None
sensorType = None # ==actuator
resetVal = None
depMapFile = 'metadata/dependency_map.json'
def __init__(self, minLatency):
# minLatency(datetime) ->
self.minLatency = minLatency
self.bdm = BDWrapper()
# depMap = pickle.load(open(self.depMapFile, 'rb'))
with open(self.depMapFile, 'rb') as fp:
depMap = json.load(fp)
if self.uuid in depMap.keys():
for depUuid in depMap[self.uuid]:
self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
self.affectedDependencyDict[depUuid] = timedelta(minutes=10)
@abstractmethod
def set_value(self, val, tp):
self.controlFlag = True
if self.inputType.validate(val):
self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
else:
print "Failed to validate a value of " + self.zone + '\'s Common Setpoint to ' + str(
val)
@abstractmethod
def get_value(self, beginTime, endTime):
return self.bdm.get_sensor_ts(self.uuid, self.sensorType, beginTime,
endTime)
@abstractmethod #Should this be abm?
def reset_value(self, val, tp):
self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
self.controlFlag = False
def get_latest_value(self, now):
result = self.bdm.get_sensor_ts(self.uuid, self.sensorType,
now - timedelta(hours=6),
now + timedelta(minutes=10))
if result.empty:
return None
else:
return result.tail(1).values[0], result.tail(1).index[0]
def get_second_latest_value(self, now):
result = self.bdm.get_sensor_ts(self.uuid, self.sensorType,
now - timedelta(hours=6),
now + timedelta(minutes=10))
if len(result) <= 1:
return None
else:
return result.tail(2).values[1], result.tail(2).index[1]
def check_control_flag(self):
return self.controlFlag
def check_dependency(self, commDict, setTime):
if (commDict['set_time'] >
setTime - self.minLatency) and commDict['uuid'] == self.uuid:
return True
else:
return False
def get_dependency(self, uuid):
if uuid in self.affectingDependencyDict.keys():
return self.affectingDependencyDict[uuid]
elif uuid in self.affectedDependencyDict.keys():
return self.affectedDependencyDict[uuid]
else:
return None
def get_dependent_actu_list(self):
return self.affectingDependencyDict.keys(
) + self.affectedDependencyDict.keys()
def get_longest_dependency(self):
return max(max(self.affectedDependencyDict.values()),
max(self.affectingDependencyDict.values()))
def validate_input(self, given):
return self.inputType.validate(given)
class Actuator(object): __metaclass__ = ABCMeta minLatency = None # 0 minimum inputType = None # should be selected among above type classes #lowerActuators = list() #higherActuators = list() affectingDependencyDict = dict() # values of this dict are each actuator's minLatency affectedDependencyDict = dict() # values of this dict are this actuator's minLatency controlFlag = False uuid = None name = None bdm = None sensorType = None # ==actuator resetVal = None depMapFile = 'metadata/dependency_map.json' def __init__(self, minLatency): # minLatency(datetime) -> self.minLatency = minLatency self.bdm = BDWrapper() # depMap = pickle.load(open(self.depMapFile, 'rb')) with open(self.depMapFile,'rb') as fp: depMap = json.load(fp) if self.uuid in depMap.keys(): for depUuid in depMap[self.uuid]: self.affectingDependencyDict[depUuid] = timedelta(minutes=10) self.affectedDependencyDict[depUuid] = timedelta(minutes=10) @abstractmethod def set_value(self, val, tp): self.controlFlag = True if self.inputType.validate(val): self.bdm.set_sensor(self.uuid, self.sensorType, tp, val) else: print "Failed to validate a value of " + self.zone + '\'s Common Setpoint to ' + str(val) @abstractmethod def get_value(self, beginTime, endTime): return self.bdm.get_sensor_ts(self.uuid, self.sensorType, beginTime, endTime) @abstractmethod #Should this be abm? def reset_value(self, val, tp): self.bdm.set_sensor(self.uuid, self.sensorType, tp, val) self.controlFlag = False def get_latest_value(self, now): result = self.bdm.get_sensor_ts(self.uuid, self.sensorType, now-timedelta(hours=6), now+timedelta(minutes=10)) if result.empty: return None else: return result.tail(1).values[0], result.tail(1).index[0] def get_second_latest_value(self,now): result = self.bdm.get_sensor_ts(self.uuid, self.sensorType, now-timedelta(hours=6), now+timedelta(minutes=10)) if len(result)<=1: return None else: return result.tail(2).values[1], result.tail(2).index[1] def check_control_flag(self): return self.controlFlag def check_dependency(self, commDict, setTime): if (commDict['set_time']> setTime-self.minLatency) and commDict['uuid']==self.uuid: return True else: return False def get_dependency(self, uuid): if uuid in self.affectingDependencyDict.keys(): return self.affectingDependencyDict[uuid] elif uuid in self.affectedDependencyDict.keys(): return self.affectedDependencyDict[uuid] else: return None def get_dependent_actu_list(self): return self.affectingDependencyDict.keys() + self.affectedDependencyDict.keys() def get_longest_dependency(self): return max(max(self.affectedDependencyDict.values()),max(self.affectingDependencyDict.values())) def validate_input(self, given): return self.inputType.validate(given)