def __init__(self, minLatency):
     # minLatency(datetime) ->
     self.minLatency = minLatency
     self.bdm = BDWrapper()
     #		depMap = pickle.load(open(self.depMapFile, 'rb'))
     with open(self.depMapFile, 'rb') as fp:
         depMap = json.load(fp)
     if self.uuid in depMap.keys():
         for depUuid in depMap[self.uuid]:
             self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
             self.affectedDependencyDict[depUuid] = timedelta(minutes=10)
class CommonSetpoint(Actuator):
	bdm = BDWrapper()
	zone = None
	template = None

#TODO: Is it okay to set minVal and maxVal arbitrarily outside this class?
	def __init__ (self, name, uuid, minVal, maxVal, zone):
		self.name = name
		self.uuid = uuid
		super(CommonSetpoint, self).__init__(timedelta(minutes=10))
		self.inputType = master_actuator.TempType(minVal, maxVal)
		self.zone = zone
		self.template = actuNames.commonSetpoint
		self.sensorType = 'PresentValue'

	def set_value(self, val, tp):
		if val=='ZT':
			ztuuid = self.bdm.get_sensor_uuids({'room':self.zone, 'template':sensorNames.zoneTemperature})[0]
			now = datetime.now()
			currZT = self.bdm.get_sensor_ts(ztuuid, 'PresentValue', now-timedelta(hours=1), now).tail().tolist()[0]
			val = currZT
		else:
			pass
		super(CommonSetpoint, self).set_value(val, tp+timedelta(minutes=1))
	
	def get_value(self, beginTime, endTime):
		return super(CommonSetpoint, self).get_value(beginTime, endTime)

	def reset_value(self, val, tp):
		super(CommonSetpoint, self).reset_value(val, tp)
	
	#TODO: Do I need this?
	def set_reset_value(self, resetVal=-1):
		self.resetVal = resetVal
예제 #3
0
	def __init__(self):
		self.actuNames = ActuatorNames()
		self.sensorNames = SensorNames()
		self.bdm = BDWrapper()
		self.expLogColl = CollectionWrapper('experience_log')
		#self.zonelist = self.csv2list('metadata/partialzonelist.csv')
		self.zonelist = self.csv2list('metadata/zonelist.csv')
		self.feater = FeatureExtractor()
		self.clust = Clusterer()
	def __init__(self, minLatency):
# minLatency(datetime) ->
		self.minLatency = minLatency
		self.bdm = BDWrapper()
#		depMap = pickle.load(open(self.depMapFile, 'rb'))
		with open(self.depMapFile,'rb') as fp:
			depMap = json.load(fp)
		if self.uuid in depMap.keys():
			for depUuid in depMap[self.uuid]:
				self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
				self.affectedDependencyDict[depUuid] = timedelta(minutes=10)
예제 #5
0
	def __init__(self):
		self.ztTrackZoneList = list()
		self.ntpClient = ntplib.NTPClient()
		self.statColl = CollectionWrapper('status')
		self.expLogColl = CollectionWrapper('experiment_log')
		self.ntpActivateTime = self.dummyBeginTime
		logging.basicConfig(filename='log/debug'+datetime.now().isoformat()[0:-7].replace(':','_') + '.log',level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s')
		#logger = logging.getLogger()
		logging.debug('Quiver initialization')
		self.bdm = BDWrapper()
		self.update_time_offset()
		self.zonelist = basic.csv2list('metadata/zonelist.csv')
		self.depMapFile = 'metadata/dependency_map.json'
	#	requests.packages.urllib3.disable_warnings()

		# Create pid file for monitoring
		pid = str(os.getpid())
예제 #6
0
class Quiver:
	ntpURL = 'ntp.ucsd.edu'
	timeOffset = timedelta(0)
	ntpClient = None
	actuDict= dict()
	actuNames = ActuatorNames()
	futureCommColl = None 	# This is a collection for future command sequence. If some of the commands are issued, they are removed from here.
	expLogColl = None	 		# This is a collection for log of control. If a command is issued, it is added to here with relevant information.
	statColl = None		# This is a collection for rollback. If a command is issued, its corresponding rollback command is added here.
	relinquishVal = -1
	ambulanceConn = None
	ntpActivateTime = None
	dummyBeginTime = datetime(2000,1,1)
	dummyEndTime = datetime(2030,12,31,0,0,0)
	bdm = None
	ackLatency = timedelta(minutes=10)
	statusExpiration = timedelta(hours=24)
	zonelist = None
	logger = None
	ztTrackZoneList = None

	def __init__(self):
		self.ztTrackZoneList = list()
		self.ntpClient = ntplib.NTPClient()
		self.statColl = CollectionWrapper('status')
		self.expLogColl = CollectionWrapper('experiment_log')
		self.ntpActivateTime = self.dummyBeginTime
		logging.basicConfig(filename='log/debug'+datetime.now().isoformat()[0:-7].replace(':','_') + '.log',level=logging.DEBUG, format='%(asctime)s [%(levelname)s] %(message)s')
		#logger = logging.getLogger()
		logging.debug('Quiver initialization')
		self.bdm = BDWrapper()
		self.update_time_offset()
		self.zonelist = basic.csv2list('metadata/zonelist.csv')
		self.depMapFile = 'metadata/dependency_map.json'
	#	requests.packages.urllib3.disable_warnings()

		# Create pid file for monitoring
		pid = str(os.getpid())
		#pidfile = "C:\\temp\\quiver.pid"
#		if os.path.isfile(pidfile):
#			print "%s already exists, exiting" %pidfile
#			sys.exit()
#		else:
#			file(pidfile, 'w').write(pid)
#TODO: Above should be enabled in final version

	def __del__(self):
		pass

	# Construct dependency map for all points
	# Run this when you run this for the fisrt time
	# TODO: For now, it finds all the points in the corresponding zone.
	def init_dependency_map(self):
		depZoneMap = defaultdict(list)
		depUuidMap = defaultdict(list)
		for zone in self.zonelist:
			for actuType in self.actuNames.nameList:
				try:
					uuid = self.get_actuator_uuid(zone, actuType)
					depZoneMap[zone].append(uuid)
				except QRError as e:
					print "There is no sensor corresponding to ", zone, actuType
		for zone, depList in depZoneMap.iteritems():
			for uuid in depList:
				for depUuid in depList:
					if uuid != depUuid:
						depUuidMap[uuid].append(depUuid)

		with open(self.depMapFile,'w') as fp:
			json.dump(depUuidMap, fp)

	def notify_systemfault(self):
		content = "Quiver control system bas been down at " + self.now().isoformat()
		self.notify_email(content)

	def notify_email(self, content):
		server = smtplib.SMTP(emailauth.smtpURL)
		msg = MIMEText('"'+content+'"')
		msg['Subject']='Alert: Quiver is down'
		msg['From'] = emailauth.fromaddr
		msg['To'] = ",".join(emailauth.toaddrs)
		server.starttls()
		server.login(emailauth.username, emailauth.password)
		server.sendmail(emailauth.fromaddr, emailauth.toaddrs, msg.as_string())
		server.quit()

# This is for NTP with UCSD BMS.
# TODO: It should be done in OS itself. Any suggestion?
	def update_time_offset(self):
		ntpRequest = self.ntpClient.request(self.ntpURL)
#		ntpRequest.tx_time
		ntpTime = datetime.strptime(time.ctime(ntpRequest.tx_time), "%a %b %d %H:%M:%S %Y")
		self.timeOffset = ntpTime - datetime.now()
		return ntpTime
	
	# Currently, zone and actuator type is the easiest information for applications to use, so I use it to get uuid, but can be generalized.
	def get_actuator_uuid(self, zone=None, actuType=None):
		context = dict()
		if zone != None:
			context['room']=zone
		if actuType != None:
			context['template']=actuType
		uuids = self.bdm.get_sensor_uuids(context)
		if len(uuids)>1:
			raise QRError('Many uuids are found', context)
		elif len(uuids)==0:
			raise QRError('No uuid is found', context)
		else:
			return uuids[0]
	
	def get_actuator_name(self,zone=None,actuType=None):
		context = dict()
		if zone != None:
			context['room']=zone
		if actuType != None:
			context['template']=actuType
		uuids = self.bdm.get_sensor_names(context)
		if len(uuids)>1:
			raise QRError('Many uuids are found', context)
		elif len(uuids)==0:
			raise QRError('No uuid is found', context)
		else:
			return uuids[0]

	def load_reset_seq(self, endTime):
		query = {'reset_time':{'$lte':endTime}}
		futureSeq = self.statColl.pop_dataframe(query)
		return futureSeq

	def actuator_exist(self, uuid):
# zone(string), actuatortype(string) -> existing?(boolean)
		if uuid in self.actuDict.keys():
			return True
		else:
			return False
			
	def rollback_to_original_setting(self):
		resetSeq = self.load_reset_seq(self.dummyEndTime)
		for row in resetSeq.iterrows():
			currTime = self.now()
			zone = row[1]['zone']
			actuType = row[1]['actuator_type']
			resetVal = row[1]['reset_value']
			uuid = row[1]['uuid']
			actuator = self.actuDict[uuid]
			actuator.reset_value(resetVal, currTime)
		self.statColl.remove_all()

	def validate_command_seq_freq(self,seq):
# seq(pd.DataFrame) -> valid?(boolean)
#TODO: This does not consider reset commands. However, it should be considered later
		baseInvalidMsg = "Test sequence is invalid because "
		for row in seq.iterrows():
			zone = row[1]['zone']
			actuType = row[1]['actuator_type']
			uuid = row[1]['uuid']
			actuator  = self.actuDict[uuid]
			minLatency = actuator.minLatency
			setTime = row[1]['set_time']
			inrangeRowsIdx = np.bitwise_and(seq['set_time']<setTime+minLatency, seq['set_time']>setTime-minLatency)
			inrangeRowsIdx = np.bitwise_and(inrangeRowsIdx, seq['uuid']==uuid)
			inrangeRowsIdx[row[0]] = False
			inrangeRows = seq[inrangeRowsIdx.values.tolist()]
			resetRows = self.statColl.load_dataframe({'uuid':uuid})
			loggedRows = self.expLogColl.load_dataframe({'$and':[{'set_time':{'$gte':setTime-actuator.minLatency}},{'uuid':uuid}]})
			inrangeRows = pd.concat([inrangeRows, resetRows, loggedRows])
			for inrangeRow in inrangeRows.iterrows():
				if inrangeRow[1]['zone']==zone and inrangeRow[1]['actuator_type']==actuType:
					print baseInvalidMsg + str(row[1]) + ' is overlapped with ' + str(inrangeRow[1])
					return row[1]
		return pd.DataFrame({})
	
	def validate_command_seq_dependency(self, seq, minExpLatency):
# seq(pd.DataFrame) -> valid?(boolean)
		#baseInvalidMsg = "Test sequence is invalid because "
		for row in seq.iterrows():
			zone = row[1]['zone']
			actuType = row[1]['actuator_type']
			uuid = row[1]['uuid']
			actuator  = self.actuDict[uuid]
			minLatency = actuator.minLatency
			setTime = row[1]['set_time']
			inrangeRowsIdx = np.bitwise_and(seq['set_time']<setTime, seq['set_time']>setTime-minExpLatency)
			inrangeRows = seq[inrangeRowsIdx.values.tolist()]
			resetRows = self.statColl.load_dataframe({})
			loggedRows = self.expLogColl.load_dataframe({'set_time':{'$gte':setTime-minExpLatency}})
			inrangeRows = pd.concat([inrangeRows, resetRows, loggedRows])
			for inrangeRow in inrangeRows.iterrows():
				if inrangeRow[1]['zone']==zone and actuator.get_dependency(inrangeRow[1]['actuator_type'])!=None:
		#			print baseInvalidMsg + str(row[1]) + ' is dependent on ' + str(inrangeRow[1])
					return row[1]
		return pd.DataFrame({})

	def static_validate(self, seq):
		invalidFreqCommand = self.validate_command_seq_freq(seq)
		if not invalidFreqCommand.empty:
			return invalidFreqCommand
		invalidDepCommand = self.validate_command_seq_dependency(seq, timedelta(minutes=5)) #TODO: This minExpLatency should be set to 1 hour later
		if not invalidDepCommand.empty:
			return invalidDepCommand
		return pd.DataFrame({})

	def validate_batch(self, seq):
		# Validate each command
		for seqRow in seq.iterrows():
			zone = seqRow[1]['zone']
			actuType = seqRow[1]['actuator_type']
			setVal = seqRow[1]['set_value']
			uuid = self.get_actuator_uuid(zone, actuType)
			name= self.get_actuator_name(zone, actuType)
			seqRow[1]['uuid'] = uuid
			seqRow[1]['name'] = name
			
			if not uuid in self.actuDict.keys():
				actuator = metaactuators.make_actuator(uuid, name, zone, actuType)
				if actuator == None:
					logging.error("Incorrect actuator type: %s", actuType)
					raise QRError('Incorrect actuator type', actuType)
				self.actuDict[uuid] = actuator
			actuator = self.actuDict[uuid]

			# Validation 1: Check input range
			if not actuator.validate_input(setVal):
				logging.error("Input value is not in correct range: %s", repr(seqRow[1]))
				raise QRError("Input value is not in correct range", seqRow[1])

			# Validation 2: Check if there is a dependent command in given batch
			for otherSeqRow in seq.iterrows():
				if seqRow[0]!=otherSeqRow[0]:
					if actuator.check_dependency(otherSeqRow[1], self.now()):
						logging.error("A command is dependent on a command in the given sequence: %s", repr(seqRow[1]) + repr(otherSeqRow[1]))
						raise QRError("A command is dependent on a command in the given sequence", repr(seqRow[1]) + repr(otherSeqRow[1]))
					elif otherSeqRow[1]['uuid'] == uuid:
						logging.error('A command has same target equipment with another: %s', repr(seqRow[1])+ repr(otherSeqRow[1]))
						raise QRError('A command has same target equipment with another', repr(seqRow[1])+ repr(otherSeqRow[1]))

			# Validation 3: Check if there is a dependent command in current status
			queryDep = {'set_time':{'$gte':self.now()-actuator.minLatency}}
			depCommands = self.statColl.load_dataframe(queryDep)
			for commRow in depCommands.iterrows():
				if actuator.check_dependency(commRow[1], self.now()):
					logging.error("A command is dependent on current status: %s", repr(seqRow[1])+repr(commRow[1]))
					raise QRError("A command is dependent on current status", repr(seqRow[1])+repr(commRow[1]))

			seq.loc[seqRow[0]] = seqRow[1]

	def now(self):
		currTime = datetime.now()
		currTime = currTime + self.timeOffset
		return currTime

# PRIMARY function.
	def issue_seq(self, seq):
		logging.debug('Start issuing: \n%s', repr(seq))
		self.validate_batch(seq)
		logging.debug('Validation is done')
		#TODO: Check if updated seq is returned

		for row in seq.iterrows():
			idx = row[0]
			zone = row[1]['zone']
			setVal = row[1]['set_value']
			actuType = row[1]['actuator_type']
			uuid = row[1]['uuid']
			name = row[1]['name']
			actuator = self.actuDict[uuid]
			now = self.now()
			origVal = actuator.get_latest_value(now)[0]
			seq.loc[idx, 'original_value'] = origVal
			if actuator.check_control_flag():
				query = {'uuid':uuid}
				resetVal = self.statColl.load_dataframe(query).tail(1)
				#TODO: This should become more safe. e.g., what if that is no data in statColl?
				resetVal = float(resetVal['reset_value'])
			else:
				resetVal = origVal
			seq.loc[idx, 'reset_value'] = resetVal
			setTime = self.now()
			seq.loc[idx, 'set_time'] = setTime
			if setVal == -1:
				if actuType in [self.actuNames.commonSetpoint]:
					setVal = float(self.statColl.load_dataframe({'uuid':uuid}).tail(1)['reset_value'][0])
				actuator.reset_value(setVal, setTime)
				logging.debug('Reset a value: %s by %s', actuType, str(setVal))
			else:
				actuator.set_value(setVal, setTime) #TODO: This should not work in test stage
				logging.debug('Set a value: %s by %s', actuType, str(setVal))
		self.ack_issue(seq)
		logging.debug("Logging done")


#TODO: initialIssueFlagList should be checked if is implemented correctly
	def ack_issue(self, seq):
		if len(seq)==0:
			return None
		seq.index = range(0,len(seq)) # Just to make it sure (need to remove?)
		issueFlagList = np.array([False]*len(seq))
		initialIssueFlagList = np.array([True]*len(seq))
		uploadedTimeList = list()
		resendInterval = timedelta(minutes=6)
		maxWaitTime = resendInterval * 2
		
		# Init uploadedTimeList
		for row in seq.iterrows():
			uuid = row[1]['uuid']
			actuator = self.actuDict[uuid]
			latestVal, setTime = actuator.get_latest_value(self.now())
			setVal = row[1]['set_value']
			# TODO: Think about timing here
			# What if a value is reset then temporal current value is restored?
			# Temporarily do not check the case where setVal==-1.
			if setVal=='-1':
				pass
			elif (setVal!=-1 and latestVal != setVal):
				latestVal, setTime = actuator.get_second_latest_value(self.now())
				if setVal!=-1 and latestVal != setVal:
					initialIssueFlagList[row[0]] = False
					logging.error("A command is not issued initially: \n%s", repr(row[1]))
				#raise QRError('Initial upload to BD is failed', row[1])
			uploadedTimeList.append(setTime)
		uploadedTimeList = np.array(uploadedTimeList)

		# Receive ack
		maxWaitDatetime = max(uploadedTimeList)+maxWaitTime
		ackInterval = 30 # secounds
		while maxWaitDatetime>=self.now(): 
			for row in seq.iterrows():
				idx = row[0]
				if initialIssueFlagList[idx]==False:
					continue
				if issueFlagList[idx]==True:
					continue
				uuid = row[1]['uuid']
				setVal = row[1]['set_value']
				origVal = row[1]['original_value']
				actuator = self.actuDict[uuid]

				if setVal == -1:
					ackVal = row[1]['reset_value']
				else:
					ackVal = row[1]['set_value']
				actuType = row[1]['actuator_type']
				actuator = self.actuDict[uuid]
				currT = self.now()
				currVal, newSetTime = actuator.get_latest_value(self.now())
				logging.debug("ack: uploadTime: %s, downloadTime: %s", str(uploadedTimeList[idx]), str(newSetTime))
				if (setVal!=-1 and currVal==ackVal and newSetTime!=uploadedTimeList[idx]) or (setVal==-1 and currVal!=-1):
					issueFlagList[idx] = True
					seq.loc[idx, 'set_time'] = uploadedTimeList[idx]
					logging.debug("Received a ack of a command: \n%s", repr(row[1]))
					if setVal==-1:
						#actuator.reset_value(currVal, uploadedTimeList[idx])
						pass
					continue
				now = self.now()
				if now>=uploadedTimeList[idx]+resendInterval:
					actuator.set_value(origVal, uploadedTimeList[idx])
					setTime = self.now()
					if setVal==-1:
						if actuType in [self.actuNames.commonSetpoint]:
							setVal = row[1]['reset_value']
						actuator.reset_value(setVal,setTime)
					else:
						actuator.set_value(setVal, setTime)
					logging.debug("Resend a command due to timeout: \n%s", repr(row[1]))
					uploadedTimeList[idx] = setTime
			if not (False in issueFlagList):
				break
			time.sleep(ackInterval)

		for row in seq.iterrows():
			if issueFlagList[row[0]] == True:
				self.reflect_an_issue_to_db(row[1])

#		for idx, flag in enumerate(issueFlagList):
#			if not flag:
#				raise QRError('Some commands are unable to be uploaded', seq[idx])
		if False in issueFlagList or False in initialIssueFlagList:
			logging.error('Some commands are unable to be uploaded: %s', repr(seq[np.logical_not(issueFlagList)]+seq[np.logical_not(initialIssueFlagList)]))
			raise QRError('Some commands are unable to be uploaded', seq[np.logical_not(issueFlagList)]+seq[np.logical_not(initialIssueFlagList)])

	def reflect_an_issue_to_db(self, commDict):
		zone = commDict['zone']
		setVal = commDict['set_value']
		actuType = commDict['actuator_type']
		uuid = commDict['uuid']
		name = commDict['name']
		resetVal = commDict['reset_value']
		setTime = commDict['set_time']
		origVal = commDict['original_value']
		actuator = self.actuDict[uuid]

		self.statColl.pop_dataframe({'uuid':uuid})
		statusRow = StatusRow(uuid, name, setTime=setTime, setVal=setVal, resetVal=resetVal, actuType=actuType, underControl=actuator.check_control_flag())
		self.statColl.store_row(statusRow)
		expLogRow = ExpLogRow(uuid, name, setTime=setTime, setVal=setVal, origVal=origVal)
		self.expLogColl.store_row(expLogRow)
			
	def reset_seq(self, seq):
		for row in seq.iterrows():
			#TODO: validate_in_log
			print row
			resetVal = row[1]['reset_value']
			actuType = row[1]['actuator_type']
			uuid = row[1]['uuid']
			name = row[1]['name']
			actuator = self.actuDict[uuid]
			now = self.now()
			origVal = actuator.get_value(now-timedelta(hours=1), now).tail(1)[0]
			actuator.reset_value(resetVal, resetTime)
			expLogRow = ExpLogRow(uuid, name, setTime=None, setVal=None, resetVal=now, origVal=origVal)
			print expLogRow
			self.expLogColl.store_row(expLogRow)

		if not self.issue_ack(seq, False).empty:
			raise QRError('A reset command cannot be set at BACNet', seq)

	def top_ntp(self):
		ntpLatency = timedelta(minutes=30)
		if self.ntpActivateTime<=self.now():
			self.update_time_offset()
			self.ntpActivateTime = self.now() + ntpLatency

	def system_close_common_behavior(self):
		pass

	def system_refresh(self):
		self.expLogColl.remove_all()
		self.statColl.remove_all()
	

	# rollback does not rollback to original state but rollback by "-1"
	def emergent_rollback(self):
		queryAll = {"under_control":True}
		resetQueue = self.statColl.load_dataframe(queryAll)
		resetQueue = resetQueue.sort(columns='set_time', axis='index')
		maxConcurrentResetNum = 10
		resetInterval = 10*60 # 10 minutes
		if len(resetQueue)==0:
			return None
		resetSeq= defaultdict(dict)
		#resetSeq = [[]]

		# Make actuators to reset and get earliest set time dependent on reset_queue
		earliestDepTime = self.dummyEndTime
		for row in resetQueue.iterrows():
			uuid = row[1]['uuid']
			name = row[1]['name']
			actuType = row[1]['actuator_type']
			if not uuid in self.actuDict.keys():
				actuator = metaactuators.make_actuator(uuid,name,actuType=actuType)
				if actuator == None:
					#raise QRError('Incorrect actuator type', actuType)
					print("Incorrect actuator type: ", actuType)
					continue
				self.actuDict[uuid] = actuator
			setTime = row[1]['set_time']
			dependentTime = setTime - actuator.get_longest_dependency()
			if earliestDepTime > dependentTime:
				earliestDepTime = dependentTime

		# Construct Reset Sequence
		# Filter redundant controls and align with dependency
		for row in resetQueue.iterrows():
			uuid = row[1]['uuid']
			inQFlag = False
			for l in resetSeq.values():
				if uuid in l.keys():
					inQFlag = True
			if inQFlag == True:
				continue
			actuator = self.actuDict[uuid]
			lastKey = 0
			now = self.now()
			row[1]['original_value'] = actuator.get_value(now-timedelta(hours=1), now).tail(1)[0]
			insertedFlag = False
			for k,l in resetSeq.iteritems():
				depList = actuator.get_dependent_actu_list()
				if not bool(set(depList) & set(l.keys())) and len(l)<maxConcurrentResetNum:
					resetSeq[k][uuid] = row[1].to_dict()
					insertedFlag = True
				lastKey = k
			lastKey += 1
			if not insertedFlag:
				resetSeq[lastKey][uuid] = row[1].to_dict()

		for key, stats in resetSeq.iteritems():
			oneResetSeq = pd.DataFrame()
			for uuid, stat in stats.iteritems():
				oneResetSeq = pd.concat([oneResetSeq, pd.DataFrame(stat, index=[0])])
			resetSeq[key] = oneResetSeq

#TODO: need to add checking acknowledgement
		# Rollback
		for k, l in resetSeq.iteritems():
			for key, stat in l.iterrows():
		#		stat = row[1]
				uuid = stat['uuid']
				name = stat['name']
				actuType = stat['actuator_type']
				if actuType in [self.actuNames.commonSetpoint]:
					setVal = stat['reset_value']
				else:
					setVal = -1
				
				actuator = self.actuDict[uuid]
				now = self.now()
				origVal = stat['original_value']
				setTime = self.now()
				expLogRow = ExpLogRow(uuid, name, setTime=setTime, setVal=setVal, origVal=origVal)
				actuator.reset_value(setVal, setTime)
			self.ack_issue(l)
			time.sleep(resetInterval)

	def get_currest_status(self):
		return self.statColl.load_dataframe({'under_control':True})
	
	def output_exp_log(self):
		self.expLogColl.to_csv()
	
	def add_zone_zt_tacking(self,zone):
		self.ztTrackZoneList.append(zone)

	def remove_zone_zt_tacking(self,zone):
		try:
			self.ztTrackZoneList.remove(zone)
		except QRError as e:
			raise QRError('Failed to remove zone from ztTrackZonelist: ', zone)

def resample_data(rawData, beginTime, endTime, sampleMethod):
    rawData = rawData[beginTime:endTime]
    if not beginTime in rawData.index:
        rawData[beginTime] = rawData.head(1)[0]
        rawData = rawData.sort_index()
    if not endTime in rawData.index:
        rawData[endTime] = rawData.tail(1)[0]
        rawData = rawData.sort_index()
    if sampleMethod == 'nextval':
        procData = rawData.resample('2Min', how='pad')
    return procData


bdm = BDWrapper()
sensors_dict = shelve.open('metadata/bacnet_devices.shelve', 'r')

#buildingName = 'ebu3b'
buildingName = sys.argv[1]

rawdataDir = 'rawdata/' + buildingName + '/'

naeDict = dict()
naeDict['ebu3b'] = ['505', '506']
deviceList = naeDict[buildingName]

beginTime = datetime(2016, 1, 18)
endTime = datetime(2016, 1, 25)

# Init raw set
from bd_wrapper import BDWrapper

def resample_data(rawData, beginTime, endTime, sampleMethod):
	rawData = rawData[beginTime:endTime]
	if not beginTime in rawData.index:
		rawData[beginTime] = rawData.head(1)[0]
		rawData = rawData.sort_index()
	if not endTime in rawData.index:
		rawData[endTime] = rawData.tail(1)[0]
		rawData = rawData.sort_index()
	if sampleMethod== 'nextval':
		procData = rawData.resample('2Min', how='pad')
	return procData


bdm = BDWrapper()
sensors_dict = shelve.open('metadata/bacnet_devices.shelve','r')

#buildingName = 'ebu3b'
buildingName = sys.argv[1]


rawdataDir = 'rawdata/'+buildingName+'/'

naeDict = dict()
naeDict['ebu3b'] = ['505', '506']
deviceList = naeDict[buildingName]

beginTime = datetime(2016,1,18)
endTime = datetime(2016,1,25)
예제 #9
0
class Analyzer:
	bdm = None
	expLogColl = None
	#timeGran = timedelta(minutes=5)
	timeGran = timedelta(minutes=2)
	actuNames = None
	sensorNames = None
	zonelist = None
	feater = None
	clust = None
	
	def __init__(self):
		self.actuNames = ActuatorNames()
		self.sensorNames = SensorNames()
		self.bdm = BDWrapper()
		self.expLogColl = CollectionWrapper('experience_log')
		#self.zonelist = self.csv2list('metadata/partialzonelist.csv')
		self.zonelist = self.csv2list('metadata/zonelist.csv')
		self.feater = FeatureExtractor()
		self.clust = Clusterer()
	
	def csv2list(self, filename):
		outputList = list()
		with open(filename, 'r') as fp:
			reader = csv.reader(fp, delimiter=',')
			for row in reader:
				outputList.append(row[0])
		return outputList

	def get_actuator_uuid(self, zone=None, actuType=None):
		context = dict()
		if zone != None:
			context['room']=zone
		if actuType != None:
			context['template']=actuType
		uuids = self.bdm.get_sensor_uuids(context)
		if len(uuids)>1:
			raise QRError('Many uuids are found', context)
		elif len(uuids)==0:
			raise QRError('No uuid is found', context)
		else:
			return uuids[0]

	def normalize_data_avg(self, rawData, beginTime, endTime):
		procData = pd.Series({beginTime:float(rawData[0])})
		tp = beginTime
		while tp<=endTime:
			tp = tp+self.timeGran
			leftSeries = rawData[:tp]
			if len(leftSeries)>0:
				idx = len(leftSeries)-1
				leftVal = leftSeries[idx]
				leftIdx = leftSeries.index[idx]
			else:
				leftVal = None
			rightSeries = rawData[tp:]
			if len(rightSeries)>0:
				rightVal = rightSeries[0]
				rightIdx = rightSeries.index[0]
			else:
				rightVal = None
			if rightVal==None and leftVal!=None:
				newVal = leftVal
			elif rightVal!=None and leftVal==None:
				newVal = rightVal
			elif tp==leftIdx:
				newVal = leftVal
			elif tp==rightIdx:
				newVal = rightVal
			elif rightVal!=None and leftVal!=None:
				leftDist = (tp - leftIdx).total_seconds()
				rightDist = (rightIdx - tp).total_seconds()
				newVal = (leftVal*rightDist+rightVal*leftDist)/(rightDist+leftDist)
			else:
				print "ERROR: no data found in raw data"
				newVal = None
			newData = pd.Series({tp:newVal})
			procData = procData.append(newData)
		return procData

	def normalize_data_nextval_deprecated(self, rawData, beginTime, endTime):
		procData = pd.Series({beginTime:float(rawData[0])})
		tp = beginTime
		while tp<=endTime:
			tp = tp+self.timeGran
			leftSeries = rawData[:tp]
			if len(leftSeries)>0:
				idx = len(leftSeries)-1
				leftVal = leftSeries[idx]
				leftIdx = leftSeries.index[idx]
			else:
				leftVal = None
			rightSeries = rawData[tp:]
			if len(rightSeries)>0:
				rightVal = rightSeries[0]
				rightIdx = rightSeries.index[0]
			else:
				rightVal = None

			if rightVal != None:
				newVal = rightVal
			else:
				newVal = leftVal

			newData = pd.Series({tp:newVal})
			procData = procData.append(newData)
		return procData

	def normalize_data(self, rawData, beginTime, endTime, normType):
		rawData = rawData[beginTime:endTime]
		if not beginTime in rawData.index:
			rawData[beginTime] = rawData.head(1)[0]
			rawData = rawData.sort_index()
		if not endTime in rawData.index:
			rawData[endTime] = rawData.tail(1)[0]
			rawData = rawData.sort_index()
		if normType=='nextval':
			procData = rawData.resample('2Min', fill_method='pad')
		elif normType=='avg':
			procData = rawData.resample('2Min', how='mean')
		else:
			procData = None

		return procData
		

	def receive_a_sensor(self, zone, actuType, beginTime, endTime, normType):
		print zone, actuType
		uuid = self.get_actuator_uuid(zone, actuType)
		rawData = self.bdm.get_sensor_ts(uuid, 'PresentValue', beginTime, endTime)
		if actuType!=self.actuNames.damperCommand:
			rawData = self.remove_negativeone(rawData)
		procData = self.normalize_data(rawData, beginTime, endTime, normType)
		return procData

	def receive_entire_sensors_notstore(self, beginTime, endTime, normType, exceptZoneList=[]):
		#TODO: Should be parallelized here
		dataDict = dict()
		for zone in self.zonelist:
			if not zone in exceptZoneList:
				dataDict[zone] = self.receive_zone_sensors(zone, beginTime, endTime, normType)
		return dataDict
	
	def receive_entire_sensors(self, beginTime, endTime, filename, normType, exceptZoneList=[]):
#		filename='data/'+beginTime.isoformat()[0:-7].replace(':','_') + '.pkl'
		dataDict = self.receive_entire_sensors_notstore(beginTime, endTime, normType, exceptZoneList=exceptZoneList)
		with open(filename, 'wb') as fp:
			pickle.dump(dataDict, fp)
#			json.dump(dataDict,fp)

	def clustering(self, inputData, dataDict):
		fftFeat = self.feater.get_fft_features(inputData, dataDict)
		minmaxFeat = self.feater.get_minmax_features(dataDict)
		dtwFeat = self.feater.get_dtw_features(inputData, dataDict)
		freqFeat = self.feater.get_freq_features(inputData, dataDict)
		featDict = dict()
		for zone in self.zonelist:
			featList = list()
			featList.append(fftFeat[zone])
			featList.append(minmaxFeat[zone])
			featList.append(dtwFeat[zone])
			#featList.append(freqFeat[zone])
			featDict[zone] = featList
		print featDict['RM-4132']
		return self.clust.cluster_kmeans(featDict)
	
	def remove_negativeone(self, data):
		if -1 in data.values:
			indices = np.where(data==-1)
			for idx in indices:
				data[idx] = data[idx-1]
		return data

	def receive_zone_sensors(self, zone, beginTime, endTime, normType):
		zoneDict = dict()
		for actuType in self.actuNames.nameList+self.sensorNames.nameList:
			if actuType=='Actual Supply Flow':
				pass
			try:
				uuid = self.get_actuator_uuid(zone, actuType)
			except QRError:
				continue
#			if actuType == self.actuNames.commonSetpoint:
#				wcad = self.receive_a_sensor(zone, 'Warm Cool Adjust', beginTime, endTime, normType)
#				data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
#				data = data + wcad
#				pass
			if actuType != self.actuNames.damperCommand:
				if actuType==self.actuNames.occupiedCommand:
					pass
				data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
			else:
				data = self.receive_a_sensor(zone, actuType, beginTime, endTime, normType)
			zoneDict[actuType] = data
		return zoneDict


	def store_zone_sensors(self, zone, beginTime, endTime, normType, filename):
		data = self.receive_zone_sensors(zone, beginTime, endTime, normType)
#		with open(filename, 'wb') as fp:
#			w = csv.DictWriter(fp, data.keys())
#			w.writeheader()
#			w.writerow(data)
		for key, val in data.iteritems():
			val.to_csv('rm4132.csv', header=key, mode='a')

	def store_minmax_dict(self):
		minDict = defaultdict(dict)
		maxDict = defaultdict(dict)
		beginTime = datetime(2015,2,1)
		endTime = datetime(2015,9,1)
		shortBeginTime = datetime(2015,8,1)
		shortEndTime = datetime(2015,8,2)

		for zone in self.zonelist:
			for pointType in self.actuNames.nameList+self.sensorNames.nameList:
				try:
					if pointType=='Occupied Command':
						minDict[zone][pointType] = 1
						maxDict[zone][pointType] = 3
					elif pointType=='Cooling Command':
						minDict[zone][pointType] = 0
						maxDict[zone][pointType] = 100
					elif pointType=='Cooling Command' or pointType=='Heating Command':
						minDict[zone][pointType] = 0
						maxDict[zone][pointType] = 100
					elif pointType=='Occupied Clg Min' or pointType=='Occupied Htg Flow' or pointType=='Cooling Max Flow':
						uuid = self.get_actuator_uuid(zone, pointType)
						data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
						minDict[zone][pointType] = min(data)
						maxDict[zone][pointType] = max(data)
					elif pointType=='Temp Occ Sts':
						minDict[zone][pointType] = 0
						maxDict[zone][pointType] = 1
					elif pointType=='Reheat Valve Command':
						minDict[zone][pointType] = 0
						maxDict[zone][pointType] = 100
					elif pointType=='Actual Supply Flow' or pointType=='Actual Sup Flow SP':
						uuid = self.get_actuator_uuid(zone, pointType)
						data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
						maxFlow = data[0]
						minDict[zone][pointType] = 0
						maxDict[zone][pointType] = maxFlow
					elif pointType=='Damper Position':
						minDict[zone][pointType] = 0
						maxDict[zone][pointType] = 100
					elif pointType=='Damper Command':
						uuid = self.get_actuator_uuid(zone, pointType)
						data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', shortBeginTime, shortEndTime)
						meanData = np.mean(data)
						stdData = np.std(data)
						meanAgain = np.mean(data[np.logical_and(data<=meanData+2*stdData, data>=meanData-2*stdData)])
						minDict[zone][pointType] = meanData-2*stdData
						maxDict[zone][pointType] = meanData+2*stdData
					else:
						uuid = self.get_actuator_uuid(zone, pointType)
						data = self.bdm.get_sensor_ts(uuid, 'Presentvalue', beginTime, endTime)
						minDict[zone][pointType] = min(data)
						maxDict[zone][pointType] = max(data)

				except:
					print "Something is wrong"
					pass
		with open('metadata/mindict.pkl', 'wb') as fp:
			pickle.dump(minDict, fp)
		with open('metadata/maxdict.pkl', 'wb') as fp:
			pickle.dump(maxDict, fp)
class Actuator(object):
    __metaclass__ = ABCMeta
    minLatency = None  # 0 minimum
    inputType = None  # should be selected among above type classes
    #lowerActuators = list()
    #higherActuators = list()
    affectingDependencyDict = dict(
    )  # values of this dict are each actuator's minLatency
    affectedDependencyDict = dict(
    )  # values of this dict are this actuator's minLatency
    controlFlag = False
    uuid = None
    name = None
    bdm = None
    sensorType = None  # ==actuator
    resetVal = None
    depMapFile = 'metadata/dependency_map.json'

    def __init__(self, minLatency):
        # minLatency(datetime) ->
        self.minLatency = minLatency
        self.bdm = BDWrapper()
        #		depMap = pickle.load(open(self.depMapFile, 'rb'))
        with open(self.depMapFile, 'rb') as fp:
            depMap = json.load(fp)
        if self.uuid in depMap.keys():
            for depUuid in depMap[self.uuid]:
                self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
                self.affectedDependencyDict[depUuid] = timedelta(minutes=10)

    @abstractmethod
    def set_value(self, val, tp):
        self.controlFlag = True
        if self.inputType.validate(val):
            self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
        else:
            print "Failed to validate a value of " + self.zone + '\'s Common Setpoint to ' + str(
                val)

    @abstractmethod
    def get_value(self, beginTime, endTime):
        return self.bdm.get_sensor_ts(self.uuid, self.sensorType, beginTime,
                                      endTime)

    @abstractmethod  #Should this be abm?
    def reset_value(self, val, tp):
        self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
        self.controlFlag = False

    def get_latest_value(self, now):
        result = self.bdm.get_sensor_ts(self.uuid, self.sensorType,
                                        now - timedelta(hours=6),
                                        now + timedelta(minutes=10))
        if result.empty:
            return None
        else:
            return result.tail(1).values[0], result.tail(1).index[0]

    def get_second_latest_value(self, now):
        result = self.bdm.get_sensor_ts(self.uuid, self.sensorType,
                                        now - timedelta(hours=6),
                                        now + timedelta(minutes=10))
        if len(result) <= 1:
            return None
        else:
            return result.tail(2).values[1], result.tail(2).index[1]

    def check_control_flag(self):
        return self.controlFlag

    def check_dependency(self, commDict, setTime):
        if (commDict['set_time'] >
                setTime - self.minLatency) and commDict['uuid'] == self.uuid:
            return True
        else:
            return False

    def get_dependency(self, uuid):
        if uuid in self.affectingDependencyDict.keys():
            return self.affectingDependencyDict[uuid]
        elif uuid in self.affectedDependencyDict.keys():
            return self.affectedDependencyDict[uuid]
        else:
            return None

    def get_dependent_actu_list(self):
        return self.affectingDependencyDict.keys(
        ) + self.affectedDependencyDict.keys()

    def get_longest_dependency(self):
        return max(max(self.affectedDependencyDict.values()),
                   max(self.affectingDependencyDict.values()))

    def validate_input(self, given):
        return self.inputType.validate(given)
class Actuator(object):
	__metaclass__ = ABCMeta
	minLatency = None # 0 minimum
	inputType = None # should be selected among above type classes
	#lowerActuators = list() 
	#higherActuators = list()
	affectingDependencyDict = dict() # values of this dict are each actuator's minLatency
	affectedDependencyDict = dict() # values of this dict are this actuator's minLatency
	controlFlag = False 
	uuid = None
	name = None
	bdm = None
	sensorType = None # ==actuator
	resetVal = None
	depMapFile = 'metadata/dependency_map.json'

	def __init__(self, minLatency):
# minLatency(datetime) ->
		self.minLatency = minLatency
		self.bdm = BDWrapper()
#		depMap = pickle.load(open(self.depMapFile, 'rb'))
		with open(self.depMapFile,'rb') as fp:
			depMap = json.load(fp)
		if self.uuid in depMap.keys():
			for depUuid in depMap[self.uuid]:
				self.affectingDependencyDict[depUuid] = timedelta(minutes=10)
				self.affectedDependencyDict[depUuid] = timedelta(minutes=10)

	@abstractmethod
	def set_value(self, val, tp):
		self.controlFlag = True
		if self.inputType.validate(val):
			self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
		else:
			print "Failed to validate a value of " + self.zone + '\'s Common Setpoint to ' + str(val)

	@abstractmethod
	def get_value(self, beginTime, endTime):
		return self.bdm.get_sensor_ts(self.uuid, self.sensorType, beginTime, endTime)

	@abstractmethod #Should this be abm?
	def reset_value(self, val, tp):
		self.bdm.set_sensor(self.uuid, self.sensorType, tp, val)
		self.controlFlag = False

	def get_latest_value(self, now):
		result = self.bdm.get_sensor_ts(self.uuid, self.sensorType, now-timedelta(hours=6), now+timedelta(minutes=10))
		if result.empty:
			return None
		else:
			return result.tail(1).values[0], result.tail(1).index[0]
	
	def get_second_latest_value(self,now):
		result = self.bdm.get_sensor_ts(self.uuid, self.sensorType, now-timedelta(hours=6), now+timedelta(minutes=10))
		if len(result)<=1:
			return None
		else:
			return result.tail(2).values[1], result.tail(2).index[1]
	
	def check_control_flag(self):
		return self.controlFlag

	def check_dependency(self, commDict, setTime):
		if (commDict['set_time']> setTime-self.minLatency) and commDict['uuid']==self.uuid:
			return True
		else:
			return False
		

	def get_dependency(self, uuid):
		if uuid in self.affectingDependencyDict.keys():
			return self.affectingDependencyDict[uuid]
		elif uuid in self.affectedDependencyDict.keys():
			return self.affectedDependencyDict[uuid]
		else:
			return None
	
	def get_dependent_actu_list(self):
		return self.affectingDependencyDict.keys() + self.affectedDependencyDict.keys()
	
	def get_longest_dependency(self):
		return max(max(self.affectedDependencyDict.values()),max(self.affectingDependencyDict.values()))

	def validate_input(self, given):
		return self.inputType.validate(given)