def updateStreamListDataPoints(self,fromDate=0,limit=10):
'''
May want to allow for getting more than 1000 points per stream, Etherios Limited by default... Adjust request size or ask for more if available if limit hit...
This module however should not miss any datapoints as it retrieves data since latest point sampled
Even though it may not bring all data in at once, calling multiple times will eventually get all data if it isnt coming in faster than the call rate
Ex. if calling daily, would need to verify that no more than 1000 points/day per stream would typically be generated
Requires an export rate faster than 1 point every ~1.5 minutes per stream to need multiple calls per day
'''
'''May need to optimize insert method to use SQLAlchemy core method to speed up bulk inserts... potential to be 25 times faster.. 12 minute run time with maxed point counts and 15 streams'''
endTimer=0
newDataPointCounter=0
commitFlag=0
for stream in self.streamListInfo: #for every data stream, get list of points
if fromDate <= 0:
lastPointTS = datamanager.getMostRecentTSDataPoint(stream[0],stream[1])+1 #add ms as to not query same data again
else:
lastPointTS = fromDate
startTimer = time.time() #prevent oversampling Etherios
if startTimer - endTimer < 1:#print "Fast Sample, Add Delay...",startTimer - endTimer
print "Delay Added:",startTimer - endTimer
time.sleep(1.2)
else:pass#print "Sample Time Delta:", str(startTimer - endTimer)
if lastPointTS:
streamPoints = self.getDataStreamPoints(stream[0],stream[1],startTime=lastPointTS,limit=limit)
else:
print "Last data point not found!"
streamPoints = self.getDataStreamPoints(stream[0],stream[1],limit=limit)
if streamPoints is None:
print "\tNo New Etherios data"
#Stream exists, but no data points! Should this stream be deleted from Etherios????
#OR.....Zero results depending responses data
elif len(streamPoints) == 0: #shouldnt happen
print "\tNo New Etherios data"
else:
print "Processing ",len(streamPoints)," Data Points..."
newDataPointCounter+=len(streamPoints)
if datamanager.fastaddDataPoints(dev_id=stream[0],stream_id=stream[1],pointList = streamPoints):
commitFlag = 1
'''
for p in streamPoints:
result = datamanager.getDataPoint(stream[0],stream[1],p[0],p[1])
if result is None:
commitFlag = 1
datamanager.addDataPoint(dev_id=stream[0],stream_id=stream[1],timestamp = p[0],datapoint=p[1])
newDataPointCounter+=1
#app.logger.debug("New Data Point Record: ",stream,p)
'''
if commitFlag:
datamanager.commitDB()
else:
print "\tNo new data"
endTimer = startTimer
print "Data Points Added:",newDataPointCounter
return newDataPointCounter
def getRecentDataPoints(self):
'''
Will be run periodically as scheduled task or on-demad (force update/refresh) to update new Etherios data points
could be incorporated into a live mode possibly? Or maybe handle seperately... (JS on user machine direct to cloud)
(Typically a daily run should suffice, and only ~12 minutes/day allowed with free heroku tier)
'''
#find latest data point
lastPointTS = datamanager.getMostRecentTSDataPoint()
print "LastSampleDate:",str(time.strftime('%B %d, %Y %H:%M:%S', time.localtime((float(lastPointTS)/1000))))
print "LastSampleDate:",lastPointTS
#query etherios for all since
#self.updateStreamListDataPoints(fromDate=lastPointTS)
self.updateStreamListDataPoints()
