def testNEQ(self):
#"""Test for Inequality"""
item1 = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
item2 = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
self.assertEqual(item1, item2)
item2.time = datetime.datetime.now()
self.assertReallyNotEqual(item1, item2)
item2.time = item1.time
item2.nodeId = 10
self.assertReallyNotEqual(item1, item2)
item2.nodeId = 1
item2.typeId = 10
self.assertReallyNotEqual(item1, item2)
item2.locationId = 3
item2.value = 0.0
self.assertReallyNotEqual(item1, item2)
def _serialobj(self):
"""Helper Method to provde an object to serialise"""
theItem = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
return theItem
def testCmp(self):
# """Test Compaison function
# (actually __lt__ for Py3K Comat)"""
item1 = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
item2 = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
self.assertEqual(item1, item2)
item2.time = datetime.datetime.now()
self.assertGreater(item2, item1)
def testEq(self):
#"""Test for Equality"""
item1 = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
item2 = models.Reading(time = NOW,
nodeId = 1,
typeId = 2,
locationId = 3,
value = 45.0)
self.assertEqual(item1, item2)
self.assertReallyEqual(item1, item2)
#Or Location id
item2.locationId = 10
self.assertReallyEqual(item1, item2)
def populatedata():
"""Populate our testing database with an example deployment etc"""
LOG = logging.getLogger(__name__)
LOG.setLevel(logging.DEBUG)
LOG.debug("Populating Testing Data")
#The Deployment
session = meta.Session()
#now = datetime.datetime.now()
now = datetime.datetime(2013, 01, 01, 00, 00, 00)
thedeployment = models.Deployment(id=1,
name="testing",
description="a testing deployment",
startDate=now)
session.merge(thedeployment)
session.flush()
#We also add a testing house
thehouse = models.House(id=1,
address="testing house",
deploymentId=thedeployment.id,
startDate=now)
session.merge(thehouse)
session.flush()
#Lets add two locations
thebedroom = models.Location(id=1, houseId=thehouse.id, roomId=1)
thebathroom = models.Location(id=2, houseId=thehouse.id, roomId=6)
session.merge(thebedroom)
session.merge(thebathroom)
session.flush()
#update the nodes so they have the correct locations
thenode = session.query(models.Node).filter_by(id=837).first()
if thenode is None:
thenode = models.Node(id=837)
session.add(thenode)
thenode.locationId = 1
thenode = session.query(models.Node).filter_by(id=838).first()
if thenode is None:
thenode = models.Node(id=838)
session.add(thenode)
thenode.locationId = 2
session.flush()
transaction.commit()
#Now we want to add a load of readings
thetime = now # - datetime.timedelta(days = 10)
endtime = now + datetime.timedelta(days=10)
#print "START TIME {0}".format(starttime)
transaction.commit()
#To test the Yields I also want an incomplte database
thehouse = models.House(id=2,
address="Poor House",
deploymentId=thedeployment.id,
startDate=now)
session.merge(thehouse)
#Add a couple of locations
#Lets add two locations
thebedroom = models.Location(id=3, houseId=thehouse.id, roomId=1)
session.merge(thebedroom)
thebathroom = models.Location(id=4, houseId=thehouse.id, roomId=6)
session.merge(thebathroom)
#And More Nodes
thenode = session.query(models.Node).filter_by(id=1061).first()
if thenode is None:
thenode = models.Node(id=1061)
session.add(thenode)
thenode.locationId = 3
thenode = session.query(models.Node).filter_by(id=1063).first()
if thenode is None:
thenode = models.Node(id=1063)
session.add(thenode)
thenode.locationId = 4
session.flush()
transaction.commit()
#Add Nodestates
thecount = 0.0
seqnum = -1
while thetime < endtime:
#Increment and roll over the sequence number
seqnum += 1
if seqnum > 255:
seqnum = seqnum - 255
for nid in [837, 838, 1061, 1063]:
locationid = 1
if nid == 838:
locationid = 2
elif nid == 1061:
locationid = 3
#Sample very 10 minutes (50% Yield)
if thetime.minute % 10 == 0:
continue
elif nid == 1063:
locationid = 4
#And remove every 3rd sample
if thetime.minute % 15 == 0:
continue
ns = models.NodeState(nodeId=nid,
parent=1,
time=thetime,
localtime=seqnum,
seq_num=seqnum)
session.add(ns)
reading = models.Reading(
nodeId=nid,
typeId=0,
time=thetime,
locationId=locationid,
value=18.0 + (2.0 * math.sin(thecount)),
)
session.add(reading)
#Increment the time
thetime = thetime + datetime.timedelta(minutes=5)
thecount = thecount + (3.14 / 144)
transaction.commit()
def populatedata(session):
"""Populate with some testing data"""
now = datetime.datetime.utcnow()
startdate = now - datetime.timedelta(days=14)
cutdate = now - datetime.timedelta(days=7)
enddate = now - datetime.timedelta(hours=1)
#Add A Deployment
thedeployment = models.Deployment(id=1, name="Testing Deployments")
session.add(thedeployment)
session.flush()
#And three servers (However only two are actually deployed)
theserver = models.Server(id=1, hostname="server1", baseid=41)
session.add(theserver)
theserver = models.Server(id=2, hostname="server2", baseid=42)
session.add(theserver)
theserver = models.Server(id=3, hostname="server3", baseid=44)
session.add(theserver)
session.flush()
#Add some houses
thehouse = models.House(id=1,
address="address1",
deploymentId=1,
serverid=1)
session.add(thehouse)
thehouse = models.House(id=2,
address="address2",
deploymentId=1,
serverid=1)
session.add(thehouse)
thehouse = models.House(id=3,
address="address3",
deploymentId=1,
serverid=1)
session.add(thehouse)
thehouse = models.House(id=4,
address="address4",
deploymentId=1,
serverid=2)
session.add(thehouse)
#House that is out of date
thehouse = models.House(id=5,
address="address5",
deploymentId=1,
serverid=2)
session.add(thehouse)
#House missing one node
thehouse = models.House(id=6, address="address6", deploymentId=1)
session.add(thehouse)
#Next nodes and locations (two for each house)
locidx = 1
allnodes = []
for house in range(5):
for room in range(2):
thisloc = models.Location(id=locidx,
houseId=house + 1,
roomId=room + 1)
session.add(thisloc)
nid = house * 10 + room
thisnode = models.Node(id=nid, locationId=locidx)
session.add(thisnode)
allnodes.append(nid)
session.flush()
locidx += 1
nidx = 1000
for x in range(2):
thisnode = models.Node(id=nidx)
session.add(thisnode)
nidx += 1
session.flush()
session.commit()
#STUFF FOR PUSH STATUS
currenttime = startdate
while currenttime <= cutdate:
for server in ["server1", "server2", "server3"]:
pstat = models.PushStatus(time=currenttime, hostname=server)
session.add(pstat)
currenttime += datetime.timedelta(hours=2)
session.flush()
session.commit()
while currenttime <= enddate:
for server in ["server1", "server2"]:
pstat = models.PushStatus(time=currenttime, hostname=server)
session.add(pstat)
currenttime += datetime.timedelta(hours=2)
#Add readings / nodestates every two hours for two weeks
#But stop 3 days ago
currenttime = startdate
seq_num = 1
while currenttime <= cutdate:
for nid in allnodes:
thereading = models.Reading(time=currenttime,
nodeId=nid,
typeId=0,
value=0)
thestate = models.NodeState(time=currenttime,
nodeId=nid,
seq_num=seq_num)
session.add(thereading)
session.add(thestate)
session.flush()
currenttime = currenttime + datetime.timedelta(hours=2)
seq_num += 1
session.flush()
session.commit()
logging.debug("ALL NODES {0}".format(allnodes))
#And then for all but the nodes assocated with house 1
while currenttime <= enddate:
for nid in allnodes[3:]:
thereading = models.Reading(time=currenttime,
nodeId=nid,
typeId=0,
value=0)
thestate = models.NodeState(time=currenttime,
nodeId=nid,
seq_num=seq_num)
session.add(thereading)
session.add(thestate)
session.flush()
currenttime = currenttime + datetime.timedelta(hours=2)
seq_num += 1
session.flush()
session.commit()
#STUFF FOR PUSLE COUNT
currenttime = startdate
pcount = 0
qry = session.query(models.SensorType).filter_by(name="Gas Pulse Count")
gas_sensor = qry.first()
while currenttime <= cutdate:
#Let node 10, 11 work
thereading = models.Reading(time=currenttime,
nodeId=40,
typeId=gas_sensor.id,
value=pcount)
session.add(thereading)
thereading = models.Reading(time=currenttime,
nodeId=41,
typeId=gas_sensor.id,
value=pcount)
session.add(thereading)
pcount += 1
currenttime += datetime.timedelta(hours=2)
while currenttime <= enddate:
thereading = models.Reading(time=currenttime,
nodeId=40,
typeId=gas_sensor.id,
value=pcount)
session.add(thereading)
thereading = models.Reading(time=currenttime,
nodeId=41,
typeId=gas_sensor.id,
value=100)
session.add(thereading)
pcount += 1
currenttime += datetime.timedelta(hours=2)
session.flush()
session.commit()
def reportdata(request):
"""
View that allows exporting of data.
"""
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
outDict["pgTitle"] = "Report"
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
if "submit" in request.POST:
LOG.debug("Dealing with form submission")
#Rscript -e "library(knitr); knit('$(rnwfile).Rnw')"
#subprocess.call(["ls","-l"],cwd="./cogentviewer/rscripts")
subprocess.call(["make"], cwd="./cogentviewer/rscripts")
LOG.debug("--> Script Finished")
#Yields
LOG.debug("Calclating Yields")
#houseqry = DBSession.query(models.House).filter(models.House.id != 25).filter(models.House.id != 32).filter(models.House.id != 34).filter(models.House.id != 35)
houseqry = DBSession.query(models.House).filter(models.House.id == 35)
for house in houseqry:
LOG.debug("Processing House {0}".format(house))
#We next need locations associated with this House
locIds = [x.id for x in house.locations]
LOG.debug("Location Ids {0}".format(locIds))
readingQry = DBSession.query(models.Reading).filter(
models.Reading.locationId.in_(locIds))
#LOG.debug(readingQry)
#LOG.debug(readingQry.first())
#LOG.debug(readingQry.count())
#Daily Counts
typeQry = DBSession.query(
models.SensorType).filter_by(name="Daily Count").first()
typeId = typeQry.id
sumQry = DBSession.query(models.Reading)
sumQry = sumQry.filter_by(typeId=typeId)
sumQry = sumQry.filter(models.Reading.locationId.in_(locIds))
if sumQry.first():
LOG.debug("Summary Exists for this node")
continue
dayQry = DBSession.query(
sqlalchemy.func.date(models.Reading.time),
models.Reading.nodeId,
models.Reading.locationId,
models.Reading.typeId,
sqlalchemy.func.count(models.Reading),
#sqlalchemy.func.count(models.Reading.typeId.distinct()),
)
dayQry = dayQry.filter(models.Reading.locationId.in_(locIds))
dayQry = dayQry.filter(models.Reading.typeId != typeQry.id)
dayQry = dayQry.group_by(models.Reading.nodeId)
dayQry = dayQry.group_by(models.Reading.locationId)
dayQry = dayQry.group_by(models.Reading.typeId)
dayQry = dayQry.group_by(sqlalchemy.func.date(models.Reading.time))
#print dayQry
#LOG.debug("Count Id is {0}".format(typeQry))
if dayQry.count() == 0:
LOG.warning("No Readings found")
continue
#import csv
#outfd = csv.writer(open("testfile.csv","wb"))
#outfd.writerow(["Date","Node","Location","Type","Count"])
#for item in dayQry.limit(5):
# print item
df = pandas.DataFrame(
dayQry.all(),
columns=["Date", "nodeId", "locationId", "typeId", "Count"])
#print df
#df.to_pickle("dump.pkl")
piv = pandas.pivot_table(df,
rows=["Date"],
cols=["nodeId", "locationId"],
values="Count",
aggfunc=numpy.mean)
#And add to the database as a new summary item
for item in piv.iteritems():
thisSeries = item[1]
nodeId, locId = thisSeries.name
for pair in thisSeries.iteritems():
time, value = pair
newReading = models.Reading(time=time,
nodeId=nodeId,
typeId=typeId,
locationId=locId,
value=value)
DBSession.add(newReading)
#print newReading
LOG.debug("Row {0} {1} complete".format(nodeId, locId))
DBSession.flush()
DBSession.flush()
# print "Done"
LOG.debug("Return Dict")
return outDict
def run(self):
"""Single iteration of the mainloop"""
#Wait for
print "Waiting on Data"
data = self.con.readline()
log = self.log
print "Recv ", data
if data:
session = self.Session()
log.debug("Pkt Received> {0}".format(data.strip()))
if "FOO:" in data:
now = datetime.datetime.utcnow()
pktdata = data[4:].strip().replace("'",
"\"") #Json only likes "'s
js = json.loads(pktdata)
#First Part is to update the Node / Node State
nodeid = js["nodeid"]
msg_seq = js["msg_seq"]
localtime = js["localtime"]
parent = js["parent"]
qry = session.query(models.Node).filter_by(id=nodeid)
thenode = qry.first()
if thenode is None:
thenode = models.Node(id=nodeid)
session.add(thenode)
session.flush()
#Check for duplicate packet
pktlist = self.recentpackets.get(nodeid, SequenceList())
prev = pktlist.contains(msg_seq)
if prev:
log.warning("Duplicate Packet Detected")
return
else:
pktlist.add(msg_seq)
self.recentpackets[nodeid] = pktlist
#Then we can create a nodestate
ns = models.NodeState(time=now,
nodeId=nodeid,
localtime=localtime,
seq_num=msg_seq,
parent=parent)
session.add(ns)
# ------------- WORK THROUGH READINGS ------------------
# == Stuff that needs converting ==
val = js.get("temp", None)
#Falls over on return of 0
if val:
val = self._convert_temp(val)
rdg = models.Reading(time=now,
nodeId=nodeid,
typeId=0,
locationId=thenode.locationId,
value=val)
log.debug("Temperature {0}".format(rdg))
session.add(rdg)
val = js.get("hum", None)
if val:
val = self._convert_hum(val)
rdg = models.Reading(time=now,
nodeId=nodeid,
typeId=2,
locationId=thenode.locationId,
value=val)
#log.debug("Humidity {0}".format(rdg))
session.add(rdg)
val = js.get("battery", None)
if val:
val = self._convert_batt(val)
rdg = models.Reading(time=now,
nodeId=nodeid,
typeId=6,
locationId=thenode.locationId,
value=val)
#log.debug("Battery {0}".format(rdg))
session.add(rdg)
# ======== AND SOME AUTOMAGIC CONVERTIONS
for item in READLIST:
val = js.get(item[0], None)
#log.debug("Getting item {0} = {1}".format(item,val))
if val is None:
log.warning("No Such Item")
continue
rdg = models.Reading(time=now,
nodeId=nodeid,
typeId=item[1],
locationId=thenode.locationId,
value=val)
#log.debug("Reading {0}".format(rdg))
session.add(rdg)
session.flush()
#Finally deal with any neighbors
nb = js.get("neighbors", None)
if nb:
for idx, item in enumerate(nb):
val = item["address"]
rdg = models.Reading(time=now,
nodeId=nodeid,
typeId=2000 + idx,
locationId=thenode.locationId,
value=val)
session.add(rdg)
session.flush()
try:
session.flush()
session.commit()
except sqlalchemy.exc.IntegrityError, e:
log.warning("Error saving item {0}".format(e))
session.close()