def serverstatus(request):
"""Server Status Page"""
out_dict = {}
out_dict = {}
out_dict["headLinks"] = homepage.genHeadUrls(request)
out_dict["sideLinks"] = homepage.genSideUrls(request)
the_user = homepage.getUser(request)
out_dict["user"] = the_user.username
out_dict["pgTitle"] = "Server Status"
#I want to check the server status
out = subprocess.Popen(["service", "ch-sf", "status"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
#If running
running = "start" in out[0]
if running:
out_dict["chsf"] = 0
else:
if out[1]:
out_dict["chsf"] = out[1]
else:
out_dict["chsf"] = out[0]
#And the Base Server
out = subprocess.Popen(["service", "ch-base", "status"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOG.debug(out)
# If running
running = "start" in out[0]
LOG.debug("base running{0}".format(running))
if running:
out_dict["chbase"] = 0
else:
if out[1]:
out_dict["chbase"] = out[1]
else:
out_dict["chbase"] = out[0]
try:
#check_output only in 2.7
out = subprocess.Popen(["tail", "/var/log/ch/BaseLogger.log"],
stdout=subprocess.PIPE).communicate()[0]
out_dict["logpass"] = True
out_dict["logtail"] = highlight(out, BashLexer(), HtmlFormatter())
except subprocess.CalledProcessError:
out_dict["logpass"] = False
out = subprocess.call(["ping", "cogentee.coventry.ac.uk", "-c", "4"])
out_dict["ping"] = out
out_dict["ifconfig"] = "None"
return out_dict
def admin(request):
"""
Show Admin Page
"""
out_dict = {}
out_dict["headLinks"] = homepage.genHeadUrls(request)
out_dict["sideLinks"] = homepage.genSideUrls(request)
the_user = homepage.getUser(request)
out_dict["user"] = the_user.username
out_dict["pgTitle"] = "Admin Interface"
# get list of users
the_qry = DBSession.query(models.User)
user_list = []
for item in the_qry:
user_list.append([item.username,
item.email,
request.route_url("user", id=item.id)])
out_dict["userList"] = user_list
return render_to_response('cogentviewer:templates/admin.mak',
out_dict,
request=request)
def timeseries(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["pgTitle"] = "Time Series Data"
#outDict["deployments"] =deps
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
return outDict
def user(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
thisUser = homepage.getUser(request)
outDict["user"] = thisUser.username
outDict["formalert"] = None
outDict["pgTitle"] = "Add / Edit User"
uid = request.matchdict.get("id", None)
log.debug("User Page Called. User Id >{0}<".format(uid))
if uid is None or uid == "":
log.debug("--> No User specified")
theUser = None
else:
theUser = DBSession.query(models.User).filter_by(id=uid).first()
#Deal with form
if "submit" in request.POST:
log.debug("Form Submitted")
username = request.POST.get("username")
usermail = request.POST.get("email")
userpass = request.POST.get("password")
log.debug("Form Details Name {0} Mail {1} Pass {2}".format(
username, usermail, userpass))
if theUser is None:
theUser = models.User(username=username)
DBSession.add(theUser)
#And update everything else
theUser.email = usermail
#Encode and store password
theUser.password = security.pwdContext.encrypt(userpass)
theUser.level = "root"
DBSession.flush()
outDict["formalert"] = "User Details updated"
#Update
log.debug("User Requred by Query {0}".format(theUser))
if theUser is None:
log.debug("No Such User")
outDict["userName"] = None
outDict["userMail"] = None
else:
outDict["userName"] = theUser.username
outDict["userMail"] = theUser.email
return outDict
def timeseries(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["pgTitle"] = "Time Series Data"
#outDict["deployments"] =deps
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
return outDict
def electricity(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["pgTitle"] = "Electricity Data"
#outDict["deployments"] =deps
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
return render_to_response('cogentviewer:templates/electricity.mak',
outDict,
request=request)
def status(request):
"""Show Status page"""
out_dict = {}
out_dict["headLinks"] = homepage.genHeadUrls(request)
out_dict["sideLinks"] = homepage.genSideUrls(request)
the_user = homepage.getUser(request)
out_dict["user"] = the_user.username
out_dict["pgTitle"] = "Node Status"
#out_dict["deployments"] =deps
out_dict["nodeDropdowns"] = homepage.getNodeDropdowns()
return render_to_response('cogentviewer:templates/status.mak',
out_dict,
request=request)
def editHouse(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theId = request.matchdict.get("id", None)
if theId:
outDict["pgTitle"] = "Add / Edit House"
theHouse = DBSession.query(models.House).filter_by(id=theId).first()
#print "-#"*30
#print theHouse
#print theHouse.toDict()
#outDict["theHouse"] = theHouse.toDict()
outDict["theHouse"] = theId
else:
outDict["pgTitle"] = "Add New House"
return render_to_response('cogentviewer:templates/house.mak',
outDict,
request=request)
def editHouse(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theId = request.matchdict.get("id",None)
if theId:
outDict["pgTitle"] = "Add / Edit House"
theHouse = DBSession.query(models.House).filter_by(id=theId).first()
#print "-#"*30
#print theHouse
#print theHouse.toDict()
#outDict["theHouse"] = theHouse.toDict()
outDict["theHouse"] = theId
else:
outDict["pgTitle"] = "Add New House"
return render_to_response('cogentviewer:templates/house.mak',
outDict,
request=request)
def showserver(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
outDict["pgTitle"] = "Server Status"
#populateSalford()
#Get a list of servers
servertable = []
servers = DBSession.query(models.Server)
now = datetime.datetime.utcnow()
outDict["currenttime"] = now
for server in servers:
thisrow = {"servername": server.hostname, "baseid": server.baseid}
thisrow["address"] = None #Address of server
thisrow["lastpush"] = None #When the last push happend
thisrow["laststate"] = None #When the last state was reported
thisrow["skew"] = None #What (if any clock skew there is)
thisrow["nodes"] = None
thisrow["node_state"] = None
thisrow["reportingnodes"] = None
thisrow["push_state"] = "warning" #Formatting for the pushstate
thisrow["state_state"] = None #Formatting for the nodestate
thisrow["skew_state"] = "warning" #Formatting for the skew state
rowstate = ""
#if server.houseid:
houseqry = DBSession.query(models.House).filter_by(serverid=server.id)
heartbeat_time = datetime.datetime.utcnow() - datetime.timedelta(
hours=8)
if houseqry.count() > 0:
thisrow["address"] = houseqry.count()
#Work out locations assocatated with these houses
all_locs = []
for house in houseqry:
all_locs.extend(house.locations)
print all_locs
nodecount = 0
reportingcount = 0
for loc in all_locs:
for node in loc.nodes:
#Increment the number of nodes
nodecount += 1
#Check if we have data
lastreport = DBSession.query(
models.NodeState).filter_by(nodeId=node.id).filter(
models.NodeState.time > heartbeat_time).first()
if lastreport:
reportingcount += 1
thisrow["nodes"] = nodecount
thisrow["reportingnodes"] = reportingcount
# if False:
# #thisrow["address"] = qry.address
# #Nodes associated with this house
# houselocations = qry.locations
# housenodes = []
# for loc in houselocations:
# locnodes = loc.nodes
# #if locnodes:
# for x in locnodes:
# lastreport = DBSession.query(models.NodeState).filter_by(nodeId = x.id).order_by(models.NodeState.time.desc()).first()
# if lastreport is not None:
# lasttime = lastreport.time
# nodestate = "error"
# reportdelta = now - lasttime
# if reportdelta.days < 1:
# nodestate = "info"
# if reportdelta.seconds < 8*(60*60):
# nodestate = "success"
# housenodes.append(["{1} : {2}".format(loc.id,loc.room.name, x.id), lasttime, nodestate])
# else:
# housenodes.append(["{1} : {2}".format(loc.id,loc.room.name, x.id), None, None])
# thisrow["nodes"] = housenodes
#--- Nodestate (last push) Details ----
qry = DBSession.query(func.max(
models.NodeState.time)).filter_by(parent=server.baseid)
qrytime = qry.first()
log.debug(qrytime)
#rowstate = "error"
#thisrow["laststate_days"] = None
qrytime = qrytime[0]
if qrytime:
td = now - qrytime
rowstate = "error" #Default
#if td.days >= 1:
if td.days <= 1:
rowstate = "info"
if td.seconds < 8 * (60 * 60):
rowstate = "success"
else:
qrytime = None
thisrow["laststate"] = qrytime
thisrow["state_state"] = rowstate
#Last Push State
#qry = DBSession.query(func.max(models.PushStatus.time)).filter_by(hostname = server.hostname)
qry = DBSession.query(
models.PushStatus).filter_by(hostname=server.hostname)
qry = qry.order_by(models.PushStatus.time.desc())
result = qry.first()
if result:
thisrow["lastpush"] = result.time
#Do States
tdelta = now - result.time
push_state = "warning"
if tdelta.days < 1:
push_state = "error"
if tdelta.seconds < (60 * 60) * 2:
push_state = "success"
thisrow["push_state"] = push_state
# if tdelta.days > 1:
# thisrow["push_state"] = "error"
# elif tdelta.seconds < (60*60)*2:
# thisrow["push_state"] = "success"
skewdelta = result.time - result.localtime
# "Local Delta {0} {1} = {2} ({3})".format(result.time,
# result.localtime,
# skewdelta,
# skewdelta.days)
thisrow["skew"] = skewdelta
skew_state = "error"
if skewdelta.days < 1:
if skewdelta.seconds < 60 * 30:
skew_state = "success"
thisrow["skew_state"] = skew_state
# if ldelta.days < 1:
# if ldelta.days < 0:
# thisrow["skew_state"] = "error"
# elif ldelta.seconds < 60:
# thisrow["skew_state"] = "success"
# elif ldelta.seconds < 60*30:
# thisrow["skew_state"] = "warning"
servertable.append(thisrow)
outDict["servertable"] = servertable
outDict["serverlist"] = []
#netlist, netmap = generate_netmap()
#outDict["serverlist"] = netlist
#outDict["servermap"] = netmap
return outDict
def nodestatus(request):
log = logging.getLogger(__name__)
outdict = {}
outdict["headLinks"] = homepage.genHeadUrls(request)
outdict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outdict["user"] = theUser.username
outdict["nodeDropdowns"] = homepage.getNodeDropdowns()
outdict["pgTitle"] = "House Status"
#So we want a list of active houses
nodeqry = DBSession.query(models.Node)
nodeqry = nodeqry.order_by(models.Node.id)
nodelist = []
for node in nodeqry:
thisnode = {
"nid": node.id,
"locationId": node.locationId,
"house": None,
"room": None,
"datalocs": None,
"datatimes": None,
"datacount": None,
"datahouse": None,
}
if node.location:
thisnode["house"] = node.location.house.address
thisnode["room"] = node.location.room.name
#Check what locations we have data for
rdgqry = DBSession.query(
models.Reading.locationId,
sqlalchemy.func.max(models.Reading.time),
sqlalchemy.func.count(models.Reading.time),
).filter_by(nodeId=node.id)
rdgqry = rdgqry.group_by(models.Reading.locationId)
#log.debug("Locations are {0}".format(rdgqry.all()))
if rdgqry.count() == 0:
pass #Hacky but needs fixing
elif rdgqry.count() == 1:
thisnode["datalocs"] = rdgqry.first()[0]
thisnode["datatimes"] = rdgqry.first()[1]
thisnode["datacount"] = rdgqry.first()[2]
hseqry = DBSession.query(
models.Location).filter_by(id=rdgqry.first()[0]).first()
thisnode["datahouse"] = hseqry.house.address
else:
thisnode["datalocs"] = [x[0] for x in rdgqry]
thisnode["datatimes"] = [x[1] for x in rdgqry]
thisnode["datacount"] = [x[2] for x in rdgqry]
nodelist.append(thisnode)
outdict["nodelist"] = nodelist
return outdict
def yieldpage(request):
"""Display the Yield Homepage"""
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["pgTitle"] = "Yield Report"
#outDict["deployments"] =deps
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
#And actual page logic
summarytable = {"totalhouses":0,
"totalnodes":0,
"nodes90":0,
"nodestoday":0}
yieldtable = []
#So we first want the list of houses
houses = DBSession.query(models.House).filter(models.House.address!="Test")
#houses = DBSession.query(models.House).filter_by(address ="69 longford road")
#houses = DBSession.query(models.House).filter_by(address ="c4 59 radnormere drive")
now = datetime.datetime.utcnow()
for house in houses:
houseyield = {"house":house.address}
summarytable["totalhouses"] += 1
nodeyields = []
#We then fetch the nodes
nodeids, nodedesc = queuenodes(house.id)
yieldsum = 0.0
#for node in nodedesc:
for node in nodedesc:
#if node["nodeid"] != 65:
# continue
summarytable["totalnodes"] += 1
yieldrow = {
"nodeid": node["nodeid"],
"room": node["room"]}
#And work out yields
yieldinfo = calcyield(node["nodeid"],startdate=datetime.datetime.utcnow())
yieldrow["lasttx"] = yieldinfo[0]
#Class /Highlighting for the last transmision
if type(yieldinfo[0]) == pandas.tslib.Timestamp:
lasttx_delta = now - yieldinfo[0]
if lasttx_delta.days <= 1:
yieldrow["txclass"] = "text-success"
summarytable["nodestoday"] += 1
elif lasttx_delta.days <= 3:
yieldrow["txclass"] = "text-warning"
elif lasttx_delta.days > 3:
yieldrow["txclass"] = "text-error"
else:
yieldrow["txclass"] = "text-success"
else:
yieldrow["txclass"] = "text-info"
yieldrow["datayield"] = yieldinfo[1]
yieldrow["packetyield"] = yieldinfo[2]
if yieldinfo[1] > 90:
summarytable["nodes90"] += 1
if yieldinfo[2] != "N/A" :
yieldsum += yieldinfo[2]
nodeyields.append(yieldrow)
#avgyield = yieldsum / len(nodedesc)
avgyield = "N/A"
houseyield["data"] = nodeyields
houseyield["avg"] = avgyield
yieldtable.append(houseyield)
outDict["yieldtable"] = yieldtable
outDict["summarytable"] = summarytable
exportYield()
return outDict
def exportdata(request):
"""
View that allows exporting of data.
"""
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
outDict["pgTitle"] = "Export"
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
LOG.debug(request.POST)
if "submit" in request.POST:
LOG.debug("Dealing with form submission")
#Fetch the House Id
houseId = request.POST.get("houseId")
LOG.debug("--> House Id {0}".format(houseId))
if houseId is None or houseId == "":
LOG.warning("No House Supplied")
outDict["warnings"] = "You must supply a house"
return outDict
#houseId = 26
#else:
# houseId = int(houseId)
#Do we want to examine all rooms
allRooms = request.POST.get("allrooms") == "on"
LOG.debug("--> Do All Rooms {0}".format(allRooms))
if allRooms:
LOG.debug("--> All Rooms checked")
locids = None
else:
##Otherwise work out what room we wish to do.
LOG.debug("--> Select rooms from List")
locids = request.POST.getall("locids")
LOG.debug("--> Locations {0}".format(locids))
#Sensors to get
sensors = request.POST.getall("stype")
LOG.debug("--> Sensors {0}".format(sensors))
if sensors is None:
LOG.debug("--> Default to just Temperature sensors")
sensorlist = [0]
else:
#for sensor in sensors:
LOG.debug("--> Processing Sensor Types")
sensorlist = []
for sensor in sensors:
theQry = DBSession.query(models.SensorType).filter_by(name=sensor).first()
LOG.debug("--> --> Processing Type {0} = {1}".format(sensor,theQry))
if theQry:
sensorlist.append(int(theQry.id))
#Append Pulse outputs
if "Power" in sensors:
theQry = DBSession.query(models.SensorType).filter_by(name="Opti Smart Count").first()
if theQry:
sensorlist.append(int(theQry.id))
LOG.debug("--> Final Sensor List {0}".format(sensorlist))
#Aggregate
aggregate = request.POST.get("aggregate")
LOG.debug("--> Aggregate {0}".format(aggregate))
if aggregate == "aggregate5":
aggregate = "5Min"
elif aggregate == "aggregate15":
aggregate = "15Min"
elif aggregate == "aggregate30":
aggregate = "30Min"
elif aggregate == "aggregateHour":
aggregate = "1H"
elif aggregate == "aggregateDaily":
aggregate = "1D"
else:
aggregate = None
#Aggregate parameters
aggparam = request.POST.getall("agg-param")
LOG.debug("--> Aggregate Parameters {0}".format(aggparam))
#Start and Stop dates
sdate = request.POST.get("startdate")
edate = request.POST.get("stopdate")
LOG.debug("start {0} Stop {1}".format(sdate,edate))
startdate = None
enddate = None
if sdate:
startdate = dateutil.parser.parse(sdate,dayfirst=True)
if edate:
enddate = dateutil.parser.parse(edate,dayfirst=True)
LOG.debug("Start {0} End {1}".format(startdate,enddate))
interpolate = request.POST.get("interp")
LOG.debug("--> Interpolate {0}".format(interpolate))
#Try the string export
LOG.debug("--> Setup Download")
#theDf = processExport(houseId
theDf = processExport(houseId=houseId,
locationIds=locids,
typeIds=sensorlist,
aggregate=aggregate,
aggregateby=aggparam,
startDate=startdate,
endDate=enddate,
interpolate=interpolate,
)
#LOG.debug(theDf)
#LOG.debug(theDf.head())
LOG.debug("Data Returned")
if type(theDf) == dict:
outDict["warnings"] = "No valid data of this type"
return outDict
else:
request.override_renderer = "pancsv"
return {"data":theDf}
return outDict
def housedebugging(request):
log = logging.getLogger(__name__)
outdict = {}
outdict["headLinks"] = homepage.genHeadUrls(request)
outdict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outdict["user"] = theUser.username
outdict["nodeDropdowns"] = homepage.getNodeDropdowns()
outdict["pgTitle"] = "House Status"
#So we want a list of active houses
theid = request.matchdict.get("id")
if theid == "":
theid = 1
else:
theid = int(theid)
log.debug("Given Id >{0}< {1}".format(theid,type(theid)))
#House
thehouse = DBSession.query(models.House).filter_by(id=theid).first()
log.debug("House is {0}".format(thehouse))
outdict["house"] = thehouse.address
#Get Locations associated with this house
locs = [[x.id,x.room.name] for x in thehouse.locations]
outdict["locs"] = locs
regnodes = []
allnodes = []
for loc in thehouse.locations:
outlist = {"location": loc.id,
"room" : loc.room.name}
nodelist = [x.id for x in loc.nodes]
allnodes.extend(nodelist)
outlist["registered"] = ",".join([str(x) for x in nodelist])
qry = DBSession.query(models.Reading).filter_by(locationId = loc.id)
qry = qry.group_by(models.Reading.nodeId)
outlist["data"] = ",".join(str(x.nodeId) for x in qry)
regnodes.append(outlist)
outdict["regnodes"] = regnodes
#Nodes that have data but are not registered to this house
locids = [x.id for x in thehouse.locations]
qry = DBSession.query(models.Reading).filter(models.Reading.locationId.in_(locids))
qry = qry.group_by(models.Reading.nodeId)
datanodes = qry.all()
unreg = []
for item in datanodes:
qry = DBSession.query(models.Node).filter_by(id = item.nodeId)
thisnode = qry.first()
if thisnode.locationId != item.locationId:
unreg.append([thisnode.id,
"({0}) {1} {2}".format(thisnode.location.id,
thisnode.location.house.address,
thisnode.location.room.name)])
outdict["unreg"] = unreg
#Finally, where are nodes that are linked here reporting data to
reglocs = []
for item in allnodes:
thisitem = {"nid": item,
"currentloc":None,
"dataloc":None}
qry = DBSession.query(models.Node).filter_by(id = item)
node = qry.first()
thisitem["currentloc"] = "({0}) {1} {2}".format(node.location.id,
node.location.house.address,
node.location.room.name)
#And work out where this node should be reporting to
datalocs = []
qry = DBSession.query(models.Reading).filter_by(nodeId = item)
qry = qry.group_by(models.Reading.locationId)
for d_item in qry:
datalocs.append("({0}) {1} {2}".format(d_item.location.id,
d_item.location.house.address,
d_item.location.room.name))
thisitem["dataloc"] = datalocs
reglocs.append(thisitem)
outdict["reglocs"] = reglocs
return outdict
def housedebugging(request):
log = logging.getLogger(__name__)
outdict = {}
outdict["headLinks"] = homepage.genHeadUrls(request)
outdict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outdict["user"] = theUser.username
outdict["nodeDropdowns"] = homepage.getNodeDropdowns()
outdict["pgTitle"] = "House Status"
#So we want a list of active houses
theid = request.matchdict.get("id")
if theid == "":
theid = 1
else:
theid = int(theid)
log.debug("Given Id >{0}< {1}".format(theid, type(theid)))
#House
thehouse = DBSession.query(models.House).filter_by(id=theid).first()
log.debug("House is {0}".format(thehouse))
outdict["house"] = thehouse.address
#Get Locations associated with this house
locs = [[x.id, x.room.name] for x in thehouse.locations]
outdict["locs"] = locs
regnodes = []
allnodes = []
for loc in thehouse.locations:
outlist = {"location": loc.id, "room": loc.room.name}
nodelist = [x.id for x in loc.nodes]
allnodes.extend(nodelist)
outlist["registered"] = ",".join([str(x) for x in nodelist])
qry = DBSession.query(models.Reading).filter_by(locationId=loc.id)
qry = qry.group_by(models.Reading.nodeId)
outlist["data"] = ",".join(str(x.nodeId) for x in qry)
regnodes.append(outlist)
outdict["regnodes"] = regnodes
#Nodes that have data but are not registered to this house
locids = [x.id for x in thehouse.locations]
qry = DBSession.query(models.Reading).filter(
models.Reading.locationId.in_(locids))
qry = qry.group_by(models.Reading.nodeId)
datanodes = qry.all()
unreg = []
for item in datanodes:
qry = DBSession.query(models.Node).filter_by(id=item.nodeId)
thisnode = qry.first()
if thisnode.locationId != item.locationId:
unreg.append([
thisnode.id,
"({0}) {1} {2}".format(thisnode.location.id,
thisnode.location.house.address,
thisnode.location.room.name)
])
outdict["unreg"] = unreg
#Finally, where are nodes that are linked here reporting data to
reglocs = []
for item in allnodes:
thisitem = {"nid": item, "currentloc": None, "dataloc": None}
qry = DBSession.query(models.Node).filter_by(id=item)
node = qry.first()
thisitem["currentloc"] = "({0}) {1} {2}".format(
node.location.id, node.location.house.address,
node.location.room.name)
#And work out where this node should be reporting to
datalocs = []
qry = DBSession.query(models.Reading).filter_by(nodeId=item)
qry = qry.group_by(models.Reading.locationId)
for d_item in qry:
datalocs.append("({0}) {1} {2}".format(
d_item.location.id, d_item.location.house.address,
d_item.location.room.name))
thisitem["dataloc"] = datalocs
reglocs.append(thisitem)
outdict["reglocs"] = reglocs
return outdict
def reportdata(request):
"""
View that allows exporting of data.
"""
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
outDict["pgTitle"] = "Report"
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
if "submit" in request.POST:
LOG.debug("Dealing with form submission")
#Rscript -e "library(knitr); knit('$(rnwfile).Rnw')"
#subprocess.call(["ls","-l"],cwd="./cogentviewer/rscripts")
subprocess.call(["make"], cwd="./cogentviewer/rscripts")
LOG.debug("--> Script Finished")
#Yields
LOG.debug("Calclating Yields")
#houseqry = DBSession.query(models.House).filter(models.House.id != 25).filter(models.House.id != 32).filter(models.House.id != 34).filter(models.House.id != 35)
houseqry = DBSession.query(models.House).filter(models.House.id == 35)
for house in houseqry:
LOG.debug("Processing House {0}".format(house))
#We next need locations associated with this House
locIds = [x.id for x in house.locations]
LOG.debug("Location Ids {0}".format(locIds))
readingQry = DBSession.query(models.Reading).filter(
models.Reading.locationId.in_(locIds))
#LOG.debug(readingQry)
#LOG.debug(readingQry.first())
#LOG.debug(readingQry.count())
#Daily Counts
typeQry = DBSession.query(
models.SensorType).filter_by(name="Daily Count").first()
typeId = typeQry.id
sumQry = DBSession.query(models.Reading)
sumQry = sumQry.filter_by(typeId=typeId)
sumQry = sumQry.filter(models.Reading.locationId.in_(locIds))
if sumQry.first():
LOG.debug("Summary Exists for this node")
continue
dayQry = DBSession.query(
sqlalchemy.func.date(models.Reading.time),
models.Reading.nodeId,
models.Reading.locationId,
models.Reading.typeId,
sqlalchemy.func.count(models.Reading),
#sqlalchemy.func.count(models.Reading.typeId.distinct()),
)
dayQry = dayQry.filter(models.Reading.locationId.in_(locIds))
dayQry = dayQry.filter(models.Reading.typeId != typeQry.id)
dayQry = dayQry.group_by(models.Reading.nodeId)
dayQry = dayQry.group_by(models.Reading.locationId)
dayQry = dayQry.group_by(models.Reading.typeId)
dayQry = dayQry.group_by(sqlalchemy.func.date(models.Reading.time))
#print dayQry
#LOG.debug("Count Id is {0}".format(typeQry))
if dayQry.count() == 0:
LOG.warning("No Readings found")
continue
#import csv
#outfd = csv.writer(open("testfile.csv","wb"))
#outfd.writerow(["Date","Node","Location","Type","Count"])
#for item in dayQry.limit(5):
# print item
df = pandas.DataFrame(
dayQry.all(),
columns=["Date", "nodeId", "locationId", "typeId", "Count"])
#print df
#df.to_pickle("dump.pkl")
piv = pandas.pivot_table(df,
rows=["Date"],
cols=["nodeId", "locationId"],
values="Count",
aggfunc=numpy.mean)
#And add to the database as a new summary item
for item in piv.iteritems():
thisSeries = item[1]
nodeId, locId = thisSeries.name
for pair in thisSeries.iteritems():
time, value = pair
newReading = models.Reading(time=time,
nodeId=nodeId,
typeId=typeId,
locationId=locId,
value=value)
DBSession.add(newReading)
#print newReading
LOG.debug("Row {0} {1} complete".format(nodeId, locId))
DBSession.flush()
DBSession.flush()
# print "Done"
LOG.debug("Return Dict")
return outDict
def showserver(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
outDict["pgTitle"] = "Server Status"
# populateSalford()
# Get a list of servers
servertable = []
servers = DBSession.query(models.Server)
now = datetime.datetime.utcnow()
outDict["currenttime"] = now
for server in servers:
thisrow = {"servername": server.hostname, "baseid": server.baseid}
thisrow["address"] = None # Address of server
thisrow["lastpush"] = None # When the last push happend
thisrow["laststate"] = None # When the last state was reported
thisrow["skew"] = None # What (if any clock skew there is)
thisrow["nodes"] = None
thisrow["node_state"] = None
thisrow["reportingnodes"] = None
thisrow["push_state"] = "warning" # Formatting for the pushstate
thisrow["state_state"] = None # Formatting for the nodestate
thisrow["skew_state"] = "warning" # Formatting for the skew state
rowstate = ""
# if server.houseid:
houseqry = DBSession.query(models.House).filter_by(serverid=server.id)
heartbeat_time = datetime.datetime.utcnow() - datetime.timedelta(hours=8)
if houseqry.count() > 0:
thisrow["address"] = houseqry.count()
# Work out locations assocatated with these houses
all_locs = []
for house in houseqry:
all_locs.extend(house.locations)
print all_locs
nodecount = 0
reportingcount = 0
for loc in all_locs:
for node in loc.nodes:
# Increment the number of nodes
nodecount += 1
# Check if we have data
lastreport = (
DBSession.query(models.NodeState)
.filter_by(nodeId=node.id)
.filter(models.NodeState.time > heartbeat_time)
.first()
)
if lastreport:
reportingcount += 1
thisrow["nodes"] = nodecount
thisrow["reportingnodes"] = reportingcount
# if False:
# #thisrow["address"] = qry.address
# #Nodes associated with this house
# houselocations = qry.locations
# housenodes = []
# for loc in houselocations:
# locnodes = loc.nodes
# #if locnodes:
# for x in locnodes:
# lastreport = DBSession.query(models.NodeState).filter_by(nodeId = x.id).order_by(models.NodeState.time.desc()).first()
# if lastreport is not None:
# lasttime = lastreport.time
# nodestate = "error"
# reportdelta = now - lasttime
# if reportdelta.days < 1:
# nodestate = "info"
# if reportdelta.seconds < 8*(60*60):
# nodestate = "success"
# housenodes.append(["{1} : {2}".format(loc.id,loc.room.name, x.id), lasttime, nodestate])
# else:
# housenodes.append(["{1} : {2}".format(loc.id,loc.room.name, x.id), None, None])
# thisrow["nodes"] = housenodes
# --- Nodestate (last push) Details ----
qry = DBSession.query(func.max(models.NodeState.time)).filter_by(parent=server.baseid)
qrytime = qry.first()
log.debug(qrytime)
# rowstate = "error"
# thisrow["laststate_days"] = None
qrytime = qrytime[0]
if qrytime:
td = now - qrytime
rowstate = "error" # Default
# if td.days >= 1:
if td.days <= 1:
rowstate = "info"
if td.seconds < 8 * (60 * 60):
rowstate = "success"
else:
qrytime = None
thisrow["laststate"] = qrytime
thisrow["state_state"] = rowstate
# Last Push State
# qry = DBSession.query(func.max(models.PushStatus.time)).filter_by(hostname = server.hostname)
qry = DBSession.query(models.PushStatus).filter_by(hostname=server.hostname)
qry = qry.order_by(models.PushStatus.time.desc())
result = qry.first()
if result:
thisrow["lastpush"] = result.time
# Do States
tdelta = now - result.time
push_state = "warning"
if tdelta.days < 1:
push_state = "error"
if tdelta.seconds < (60 * 60) * 2:
push_state = "success"
thisrow["push_state"] = push_state
# if tdelta.days > 1:
# thisrow["push_state"] = "error"
# elif tdelta.seconds < (60*60)*2:
# thisrow["push_state"] = "success"
skewdelta = result.time - result.localtime
# "Local Delta {0} {1} = {2} ({3})".format(result.time,
# result.localtime,
# skewdelta,
# skewdelta.days)
thisrow["skew"] = skewdelta
skew_state = "error"
if skewdelta.days < 1:
if skewdelta.seconds < 60 * 30:
skew_state = "success"
thisrow["skew_state"] = skew_state
# if ldelta.days < 1:
# if ldelta.days < 0:
# thisrow["skew_state"] = "error"
# elif ldelta.seconds < 60:
# thisrow["skew_state"] = "success"
# elif ldelta.seconds < 60*30:
# thisrow["skew_state"] = "warning"
servertable.append(thisrow)
outDict["servertable"] = servertable
outDict["serverlist"] = []
# netlist, netmap = generate_netmap()
# outDict["serverlist"] = netlist
# outDict["servermap"] = netmap
return outDict
def node(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
outDict["pgTitle"] = "Node Data"
#outDict["deployments"] =deps
nid = request.matchdict.get("id",None)
outDict["nid"] = nid
#So we want to get the sensors attachd to this node.
theNode = DBSession.query(models.Node).filter_by(id=nid).first()
if theNode is None: #So if we have a node that doesnt exist
raise httpexp.HTTPNotFound
#Next we want the nodes location information
theLocation = theNode.location
locDetails = "No Current Location"
if theLocation is not None:
#Update
locDetails = "{0} ({1})".format(theLocation.house.address,theLocation.room.name)
outDict["locDetails"] = locDetails
#Fetch the last transmission
lastNodeState = DBSession.query(models.NodeState).filter_by(nodeId = nid).order_by(models.NodeState.time.desc()).first()
outDict["lastState"] = lastNodeState.time
#And the latest sensor readings
allReadings = DBSession.query(models.Reading).filter_by(nodeId=nid,time=lastNodeState.time)
#Length of time our graphs are up for
# buttonList = ["<a href='?duration=week' class='btn'>Last Week</a>",
# "<a href='?duration=day' class='btn'>Last Day</a>",
# "<a href='?duration=hour' class='btn'>Last Hour</a>"]
buttonList = [["year", "Last Year", "btn"],
["month", "Last Month", "btn"],
["week", "Last Week", 'btn'],
["day", "Last Day", 'btn'],
["hour", "Last Hour", 'btn'],
]
getLength = request.GET.get("duration",None)
if getLength == "year":
buttonList[0][2] = "btn btn-primary"
glength = "-1y"
elif getLength == "month":
buttonList[1][2] = "btn btn-primary"
glength = "-1m"
elif getLength == "week":
buttonList[2][2] = "btn btn-primary"
glength = "-1w"
elif getLength == "day":
buttonList[3][2] = "btn btn-primary"
glength = "-1d"
else:
buttonList[4][2] = "btn btn-primary"
glength = "-1h"
outDict["btnList"] = buttonList
outReadings = []
sensorIds = []
#Have an empyty battery Level
outDict["batLevel"] = None
for item in allReadings:
log.debug("Graph for type {0}".format(item.typeId))
sensorIds.append(item.typeId)
if item.typeId == 6:
outDict["batLevel"] = item.value
sType = item.sensorType.name
sUnits = item.sensorType.units
theURL = None
outReadings.append([sType, item.value,theURL])
outDict["allReadings"] = outReadings
return outDict
def reportdata(request):
"""
View that allows exporting of data.
"""
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
outDict["pgTitle"] = "Report"
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
if "submit" in request.POST:
LOG.debug("Dealing with form submission")
#Rscript -e "library(knitr); knit('$(rnwfile).Rnw')"
#subprocess.call(["ls","-l"],cwd="./cogentviewer/rscripts")
subprocess.call(["make"],cwd="./cogentviewer/rscripts")
LOG.debug("--> Script Finished")
#Yields
LOG.debug("Calclating Yields")
#houseqry = DBSession.query(models.House).filter(models.House.id != 25).filter(models.House.id != 32).filter(models.House.id != 34).filter(models.House.id != 35)
houseqry = DBSession.query(models.House).filter(models.House.id == 35)
for house in houseqry:
LOG.debug("Processing House {0}".format(house))
#We next need locations associated with this House
locIds = [x.id for x in house.locations]
LOG.debug("Location Ids {0}".format(locIds))
readingQry = DBSession.query(models.Reading).filter(models.Reading.locationId.in_(locIds))
#LOG.debug(readingQry)
#LOG.debug(readingQry.first())
#LOG.debug(readingQry.count())
#Daily Counts
typeQry = DBSession.query(models.SensorType).filter_by(name="Daily Count").first()
typeId = typeQry.id
sumQry = DBSession.query(models.Reading)
sumQry = sumQry.filter_by(typeId = typeId)
sumQry = sumQry.filter(models.Reading.locationId.in_(locIds))
if sumQry.first():
LOG.debug("Summary Exists for this node")
continue
dayQry = DBSession.query(sqlalchemy.func.date(models.Reading.time),
models.Reading.nodeId,
models.Reading.locationId,
models.Reading.typeId,
sqlalchemy.func.count(models.Reading),
#sqlalchemy.func.count(models.Reading.typeId.distinct()),
)
dayQry = dayQry.filter(models.Reading.locationId.in_(locIds))
dayQry = dayQry.filter(models.Reading.typeId != typeQry.id)
dayQry = dayQry.group_by(models.Reading.nodeId)
dayQry = dayQry.group_by(models.Reading.locationId)
dayQry = dayQry.group_by(models.Reading.typeId)
dayQry = dayQry.group_by(sqlalchemy.func.date(models.Reading.time))
#print dayQry
#LOG.debug("Count Id is {0}".format(typeQry))
if dayQry.count() == 0:
LOG.warning("No Readings found")
continue
#import csv
#outfd = csv.writer(open("testfile.csv","wb"))
#outfd.writerow(["Date","Node","Location","Type","Count"])
#for item in dayQry.limit(5):
# print item
df = pandas.DataFrame(dayQry.all(),columns=["Date","nodeId","locationId","typeId","Count"])
#print df
#df.to_pickle("dump.pkl")
piv = pandas.pivot_table(df,rows=["Date"],cols=["nodeId","locationId"],values="Count",aggfunc=numpy.mean)
#And add to the database as a new summary item
for item in piv.iteritems():
thisSeries = item[1]
nodeId,locId = thisSeries.name
for pair in thisSeries.iteritems():
time,value = pair
newReading = models.Reading(time=time,
nodeId=nodeId,
typeId=typeId,
locationId=locId,
value=value)
DBSession.add(newReading)
#print newReading
LOG.debug("Row {0} {1} complete".format(nodeId,locId))
DBSession.flush()
DBSession.flush()
# print "Done"
LOG.debug("Return Dict")
return outDict
def yieldpage(request):
"""Display the Yield Homepage"""
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["pgTitle"] = "Yield Report"
#outDict["deployments"] =deps
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
#And actual page logic
summarytable = {
"totalhouses": 0,
"totalnodes": 0,
"nodes90": 0,
"nodestoday": 0
}
yieldtable = []
#So we first want the list of houses
houses = DBSession.query(
models.House).filter(models.House.address != "Test")
#houses = DBSession.query(models.House).filter_by(address ="69 longford road")
#houses = DBSession.query(models.House).filter_by(address ="c4 59 radnormere drive")
now = datetime.datetime.utcnow()
for house in houses:
houseyield = {"house": house.address}
summarytable["totalhouses"] += 1
nodeyields = []
#We then fetch the nodes
nodeids, nodedesc = queuenodes(house.id)
yieldsum = 0.0
#for node in nodedesc:
for node in nodedesc:
#if node["nodeid"] != 65:
# continue
summarytable["totalnodes"] += 1
yieldrow = {"nodeid": node["nodeid"], "room": node["room"]}
#And work out yields
yieldinfo = calcyield(node["nodeid"],
startdate=datetime.datetime.utcnow())
yieldrow["lasttx"] = yieldinfo[0]
#Class /Highlighting for the last transmision
if type(yieldinfo[0]) == pandas.tslib.Timestamp:
lasttx_delta = now - yieldinfo[0]
if lasttx_delta.days <= 1:
yieldrow["txclass"] = "text-success"
summarytable["nodestoday"] += 1
elif lasttx_delta.days <= 3:
yieldrow["txclass"] = "text-warning"
elif lasttx_delta.days > 3:
yieldrow["txclass"] = "text-error"
else:
yieldrow["txclass"] = "text-success"
else:
yieldrow["txclass"] = "text-info"
yieldrow["datayield"] = yieldinfo[1]
yieldrow["packetyield"] = yieldinfo[2]
if yieldinfo[1] > 90:
summarytable["nodes90"] += 1
if yieldinfo[2] != "N/A":
yieldsum += yieldinfo[2]
nodeyields.append(yieldrow)
#avgyield = yieldsum / len(nodedesc)
avgyield = "N/A"
houseyield["data"] = nodeyields
houseyield["avg"] = avgyield
yieldtable.append(houseyield)
outDict["yieldtable"] = yieldtable
outDict["summarytable"] = summarytable
exportYield()
return outDict
def node(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outDict["user"] = theUser.username
outDict["nodeDropdowns"] = homepage.getNodeDropdowns()
outDict["pgTitle"] = "Node Data"
#outDict["deployments"] =deps
nid = request.matchdict.get("id", None)
outDict["nid"] = nid
#So we want to get the sensors attachd to this node.
theNode = DBSession.query(models.Node).filter_by(id=nid).first()
if theNode is None: #So if we have a node that doesnt exist
raise httpexp.HTTPNotFound
#Next we want the nodes location information
theLocation = theNode.location
locDetails = "No Current Location"
if theLocation is not None:
#Update
locDetails = "{0} ({1})".format(theLocation.house.address,
theLocation.room.name)
outDict["locDetails"] = locDetails
#Fetch the last transmission
lastNodeState = DBSession.query(models.NodeState).filter_by(
nodeId=nid).order_by(models.NodeState.time.desc()).first()
outDict["lastState"] = lastNodeState.time
#And the latest sensor readings
allReadings = DBSession.query(models.Reading).filter_by(
nodeId=nid, time=lastNodeState.time)
#Length of time our graphs are up for
# buttonList = ["<a href='?duration=week' class='btn'>Last Week</a>",
# "<a href='?duration=day' class='btn'>Last Day</a>",
# "<a href='?duration=hour' class='btn'>Last Hour</a>"]
buttonList = [
["year", "Last Year", "btn"],
["month", "Last Month", "btn"],
["week", "Last Week", 'btn'],
["day", "Last Day", 'btn'],
["hour", "Last Hour", 'btn'],
]
getLength = request.GET.get("duration", None)
if getLength == "year":
buttonList[0][2] = "btn btn-primary"
glength = "-1y"
elif getLength == "month":
buttonList[1][2] = "btn btn-primary"
glength = "-1m"
elif getLength == "week":
buttonList[2][2] = "btn btn-primary"
glength = "-1w"
elif getLength == "day":
buttonList[3][2] = "btn btn-primary"
glength = "-1d"
else:
buttonList[4][2] = "btn btn-primary"
glength = "-1h"
outDict["btnList"] = buttonList
outReadings = []
sensorIds = []
#Have an empyty battery Level
outDict["batLevel"] = None
for item in allReadings:
log.debug("Graph for type {0}".format(item.typeId))
sensorIds.append(item.typeId)
if item.typeId == 6:
outDict["batLevel"] = item.value
sType = item.sensorType.name
sUnits = item.sensorType.units
theURL = None
outReadings.append([sType, item.value, theURL])
outDict["allReadings"] = outReadings
return outDict
def nodestatus(request):
log = logging.getLogger(__name__)
outdict = {}
outdict["headLinks"] = homepage.genHeadUrls(request)
outdict["sideLinks"] = homepage.genSideUrls(request)
theUser = homepage.getUser(request)
outdict["user"] = theUser.username
outdict["nodeDropdowns"] = homepage.getNodeDropdowns()
outdict["pgTitle"] = "House Status"
#So we want a list of active houses
nodeqry = DBSession.query(models.Node)
nodeqry = nodeqry.order_by(models.Node.id)
nodelist = []
for node in nodeqry:
thisnode = {"nid": node.id,
"locationId":node.locationId,
"house":None,
"room":None,
"datalocs" : None,
"datatimes" : None,
"datacount" : None,
"datahouse" : None,
}
if node.location:
thisnode["house"] = node.location.house.address
thisnode["room"] = node.location.room.name
#Check what locations we have data for
rdgqry = DBSession.query(models.Reading.locationId,
sqlalchemy.func.max(models.Reading.time),
sqlalchemy.func.count(models.Reading.time),
).filter_by(nodeId = node.id)
rdgqry = rdgqry.group_by(models.Reading.locationId)
#log.debug("Locations are {0}".format(rdgqry.all()))
if rdgqry.count() == 0:
pass #Hacky but needs fixing
elif rdgqry.count() == 1:
thisnode["datalocs"] = rdgqry.first()[0]
thisnode["datatimes"] = rdgqry.first()[1]
thisnode["datacount"] = rdgqry.first()[2]
hseqry = DBSession.query(models.Location).filter_by(id = rdgqry.first()[0]).first()
thisnode["datahouse"] = hseqry.house.address
else:
thisnode["datalocs"] = [x[0] for x in rdgqry]
thisnode["datatimes"] = [x[1] for x in rdgqry]
thisnode["datacount"] = [x[2] for x in rdgqry]
nodelist.append(thisnode)
outdict["nodelist"] = nodelist
return outdict
def user(request):
outDict = {}
outDict["headLinks"] = homepage.genHeadUrls(request)
outDict["sideLinks"] = homepage.genSideUrls(request)
thisUser = homepage.getUser(request)
outDict["user"] = thisUser.username
outDict["formalert"] = None
outDict["pgTitle"] = "Add / Edit User"
uid = request.matchdict.get("id",None)
log.debug("User Page Called. User Id >{0}<".format(uid))
if uid is None or uid == "":
log.debug("--> No User specified")
theUser = None
else:
theUser =DBSession.query(models.User).filter_by(id=uid).first()
#Deal with form
if "submit" in request.POST:
log.debug("Form Submitted")
username = request.POST.get("username")
usermail = request.POST.get("email")
userpass = request.POST.get("password")
log.debug("Form Details Name {0} Mail {1} Pass {2}".format(username,usermail,userpass))
if theUser is None:
theUser = models.User(username = username)
DBSession.add(theUser)
#And update everything else
theUser.email = usermail
#Encode and store password
theUser.password = security.pwdContext.encrypt(userpass)
theUser.level = "root"
DBSession.flush()
outDict["formalert"] = "User Details updated"
#Update
log.debug("User Requred by Query {0}".format(theUser))
if theUser is None:
log.debug("No Such User")
outDict["userName"] = None
outDict["userMail"] = None
else:
outDict["userName"] = theUser.username
outDict["userMail"] = theUser.email
return outDict