def plotNumSensors(db):
db.create_index('date')
a = db._date.keys()
a.sort()
start = calc.datetonumber("2008-01-01 00:00:00")
start /= 100000
print start
sensAll = []
diff = a[-1] - start
print diff
for i in range(diff):
tmp = db('date') == start + i
sensAll.append(len(tmp))
fig = plt.figure()
ax = fig.add_subplot(111)
ind = numpy.arange(diff)
print sensAll
rect = ax.bar(ind, sensAll, 0.3, color = 'r')
plt.show()
def getNumActivations(db):
db.create_index('date')
a = db._date.keys()
a.sort()
start = calc.datetonumber("2008-03-17 00:00:00")
start /= 100000
dailyTotal = []
diff = a[-1] - start
print "Total number of days:" + str(diff)
for i in range(diff - 1):
print "Days:" + str(i)
tmp = db('date') == start + i
total = 0
for j in range(len(tmp)):
sens = tmp.records[j]['sensor']
date = tmp.records[j]['date']
tmp2 = (db('date') == date + 1) & (db('sensor') == sens)
if len(tmp2) == 1:
total += tmp2.records[0]['index'] - tmp.records[j]['index']
dailyTotal.append(total)
print dailyTotal
def makeBlocks(data, window, skip, startDate = "2008-03-17 00:00:00", \
endDate = "2008-03-30 00:00:00", \
sensors = [63, 62, 61, 60, 54, 53, 52, 51, 50, 44, 43, 24, \
42, 41, 34, 40, 33, 31, 32, 30, 81, 82, 83, 90]):
sd = datetime.datetime.strptime(startDate, "%Y-%m-%d %H:%M:%S")
ed = datetime.datetime.strptime(endDate, "%Y-%m-%d %H:%M:%S")
sk = datetime.timedelta(seconds=skip)
wn = datetime.timedelta(seconds=window)
secondsdiff = (ed - sd).days * 86400 + (ed - sd).seconds
numwindows = int((secondsdiff - window) / skip)
blocks = [0] * numwindows
#Iterate over all sensors
for s in sensors:
spoint = 0
epoint = 0
#Iterate over time range
for i in range(numwindows):
evalue = calc.datetonumber(sd + i * sk + wn)
svalue = calc.datetonumber(sd + i * sk)
#Find the first start point in the window
while spoint < len(data[s]):
if data[s][spoint] >= svalue:
break
spoint += 1
#Find the first end point past the window
while True:
if data[s][epoint] > evalue:
break
epoint += 1
#Check if spoint is valid
if data[s][spoint] <= evalue:
blocks[i] += epoint - spoint
return blocks
def makeBlocks(data, window, skip, startDate = "2008-03-17 00:00:00", \
endDate = "2008-03-30 00:00:00", \
sensors = [63, 62, 61, 60, 54, 53, 52, 51, 50, 44, 43, 24, \
42, 41, 34, 40, 33, 31, 32, 30, 81, 82, 83, 90]):
sd = datetime.datetime.strptime(startDate, "%Y-%m-%d %H:%M:%S")
ed = datetime.datetime.strptime(endDate, "%Y-%m-%d %H:%M:%S")
sk = datetime.timedelta(seconds = skip)
wn = datetime.timedelta(seconds = window)
secondsdiff = (ed - sd).days * 86400 + (ed - sd).seconds
numwindows = int((secondsdiff - window)/skip)
blocks = [0] * numwindows
#Iterate over all sensors
for s in sensors:
spoint = 0
epoint = 0
#Iterate over time range
for i in range(numwindows):
evalue = calc.datetonumber(sd + i * sk + wn)
svalue = calc.datetonumber(sd + i * sk)
#Find the first start point in the window
while spoint < len(data[s]):
if data[s][spoint] >= svalue:
break
spoint += 1
#Find the first end point past the window
while True:
if data[s][epoint] > evalue:
break
epoint += 1
#Check if spoint is valid
if data[s][spoint] <= evalue:
blocks[i] += epoint - spoint
return blocks
def makeCounts(data, binWidth, startDate = "2008-03-17 00:00:00", \
endDate = "2008-03-17 23:59:59", \
sensors = bbdata.allSensors):
"""construct a set of counts from the given database
"""
sd = datetime.datetime.strptime(startDate, "%Y-%m-%d %H:%M:%S")
ed = datetime.datetime.strptime(endDate, "%Y-%m-%d %H:%M:%S")
wn = datetime.timedelta(seconds = binWidth)
secondsDiff = (ed - sd).days * 86400 + (ed - sd).seconds
numbins = int(math.ceil((1.0 * secondsDiff)/binWidth))
counts = [0] * numbins
sValue = calc.datetonumber(sd)
eValue = calc.datetonumber(ed)
#Iterate over all sensors
for s in sensors:
#for s in [41]:
sIndex = 0
currentIndex = 0
t = (data["db"]("sensor") == s) & (data["db"]("date") == sd.toordinal())
currentIndex = t.records[0]['index']
currentValue = data[s][currentIndex]
while currentValue < eValue:
if currentValue > sValue:
#Replace this with something that handles multiple days
sec = currentValue - sValue
counts[int(sec/binWidth)] += 1
currentIndex += 1
currentValue = data[s][currentIndex]
return counts
def makeDB(read, write, startTime = "2010-01-01 00:00:00", \
endTime = "2010-01-01 00:10:00"):
db = Base(write)
startTime = calc.datetonumber(startTime)
endTime = calc.datetonumber(endTime)
#Day comes from day of the week. It is a number from 0 to 6.
#0 = Monday 6 = Sunday.
db.create('sensor', 'date', 'weekday', 'index', mode="override")
db.open()
allData = {}
for i in range(len(bbdata.allSensors)):
s = bbdata.allSensors[i]
data = []
print "Parsing sensor " + str(s)
try:
sString = read + "sensor" + str(s) + ".txt"
f = open(sString).readlines()
oldD = None
for timeLine in f:
tmp = timeLine.split()
tmp = tmp[1] + " " + tmp[2]
#tmp = tmp[0] + " " + tmp[1]
d = datetime.datetime.strptime(tmp, "%Y-%m-%d %H:%M:%S")
foo = calc.datetonumber(d)
if foo >= startTime and foo <= endTime:
data.append(calc.datetonumber(d))
if d.toordinal() != oldD:
#Add to database
db.insert(s, d.toordinal(), d.weekday(), len(data) - 1)
oldD = d.toordinal()
print " " + str(d)
except Exception, e:
print "Except:" + str(e)
pass
allData[s] = data
def makeDB(read, write, startTime = "2010-01-01 00:00:00", \
endTime = "2010-01-01 00:10:00"):
db = Base(write)
startTime = calc.datetonumber(startTime)
endTime = calc.datetonumber(endTime)
#Day comes from day of the week. It is a number from 0 to 6.
#0 = Monday 6 = Sunday.
db.create('sensor', 'date', 'weekday', 'index', mode="override")
db.open()
allData = {}
for i in range(len(bbdata.allSensors)):
s = bbdata.allSensors[i]
data = []
print "Parsing sensor " + str(s)
try:
sString = read + "sensor" + str(s) + ".txt"
f = open(sString).readlines()
oldD = None
for timeLine in f:
tmp = timeLine.split()
tmp = tmp[1] + " " + tmp[2]
#tmp = tmp[0] + " " + tmp[1]
d = datetime.datetime.strptime(tmp, "%Y-%m-%d %H:%M:%S")
foo = calc.datetonumber(d)
if foo >= startTime and foo <= endTime:
data.append(calc.datetonumber(d))
if d.toordinal() != oldD:
#Add to database
db.insert(s, d.toordinal(), d.weekday(), len(data) - 1)
oldD = d.toordinal()
print " " + str(d)
except Exception, e:
print "Except:" + str(e)
pass
allData[s] = data
def getdata(st, et, \
pStart = datetime.datetime.strptime("00:00:00", "%H:%M:%S"), \
pEnd = datetime.datetime.strptime("23:59:59", "%H:%M:%S"), \
vDays = [0, 1, 2, 3, 4, 5, 6], \
comp = 1, \
sens = allSensors, \
readLocation = bLocation,
compress = True):
"""getdata from a given database location.
st = start time
et = end time.
comp = how compressed (in seconds) the returned data list should be.
compression uses the compressVector method
compress = If the returned result should be as an array for each
sensor or as a single number representing the data of the
region of sensors given.
Both start and end times must be of datetime objects.
returns a numpy array.
For vDays -- 0 = Monday. 6 = Sunday.
"""
global allData
global dataLocation
#Open the database -- Do not open if already in memory.
if (not allData) or (not (readLocation == dataLocation)):
allData = dataio.loadData(readLocation + "data.dat")
dataLocation = readLocation
positions = {} #Position in array for each sensor
db = allData['db']
timeList = []
cData = [] #Data set being built.
ct = st
oneSec = datetime.timedelta(seconds = 1)
tmp = 0
#Find the starting positions for all sensors. -- Position will always be
#the index of the next sensor location past the current time.
for s in sens:
ct = st.toordinal()
while True:
tmp = (db('date') == ct) & (db('sensor') == s)
if len(tmp.records) > 0:
positions[s] = tmp.records[0]['index']
break
else:
ct += 1
if ct > et.toordinal():
#Set it to an upper limit.
positions[s] = 1000000000
break
ct = st
#Check if position isn't far enough and advance as necessary
current = calc.datetonumber(ct)
try:
while allData[s][positions[s]] < current:
positions[s] += 1
except:
positions[s] = 1000000000
while(ct <= et):
#Check if the time is valid.
if ct.weekday() in vDays:
if _validTime(ct, pStart, pEnd):
cVec = [0] * len(sens)
for i in range(len(sens)):
if positions.has_key(sens[i]):
#Convert the ct to compressed time
current = calc.datetonumber(ct)
t = positions[sens[i]]
#If the current time plus the comp time pass some real
#data, then update cVec and position
if t >= len(allData[sens[i]]) - 1:
continue
if current + comp >= allData[sens[i]][t]:
cVec[i] = 1
#print "Sensor:" + str(sens[i]) + " t:" + str(t) + " len:" + str(len(allData[sens[i]])) + " CC:" + str(current + comp) + " end:" + str(allData[sens[i]][-1])
#Find the next valid position and update.
while((current + comp > allData[sens[i]][t]) and \
(t < len(allData[sens[i]]) - 1)):
t += 1
positions[sens[i]] = t
#Determine the compressed value for cVec
if compress:
cData.append(compressVector(cVec))
timeList.append(ct)
#At the end add a second
ct += oneSec * comp
#Delete all objects
#del ad
cData = numpy.array(cData)
cData.resize(cData.shape[0], 1)
timeList = numpy.array(timeList)
timeList.resize(timeList.shape[0], 1)
return cData, timeList