def updateStreamListDataPoints(self,fromDate=0,limit=10):
'''
May want to allow for getting more than 1000 points per stream, Etherios Limited by default... Adjust request size or ask for more if available if limit hit...
This module however should not miss any datapoints as it retrieves data since latest point sampled
Even though it may not bring all data in at once, calling multiple times will eventually get all data if it isnt coming in faster than the call rate
Ex. if calling daily, would need to verify that no more than 1000 points/day per stream would typically be generated
Requires an export rate faster than 1 point every ~1.5 minutes per stream to need multiple calls per day
'''
'''May need to optimize insert method to use SQLAlchemy core method to speed up bulk inserts... potential to be 25 times faster.. 12 minute run time with maxed point counts and 15 streams'''
endTimer=0
newDataPointCounter=0
commitFlag=0
for stream in self.streamListInfo: #for every data stream, get list of points
if fromDate <= 0:
lastPointTS = datamanager.getMostRecentTSDataPoint(stream[0],stream[1])+1 #add ms as to not query same data again
else:
lastPointTS = fromDate
startTimer = time.time() #prevent oversampling Etherios
if startTimer - endTimer < 1:#print "Fast Sample, Add Delay...",startTimer - endTimer
print "Delay Added:",startTimer - endTimer
time.sleep(1.2)
else:pass#print "Sample Time Delta:", str(startTimer - endTimer)
if lastPointTS:
streamPoints = self.getDataStreamPoints(stream[0],stream[1],startTime=lastPointTS,limit=limit)
else:
print "Last data point not found!"
streamPoints = self.getDataStreamPoints(stream[0],stream[1],limit=limit)
if streamPoints is None:
print "\tNo New Etherios data"
#Stream exists, but no data points! Should this stream be deleted from Etherios????
#OR.....Zero results depending responses data
elif len(streamPoints) == 0: #shouldnt happen
print "\tNo New Etherios data"
else:
print "Processing ",len(streamPoints)," Data Points..."
newDataPointCounter+=len(streamPoints)
if datamanager.fastaddDataPoints(dev_id=stream[0],stream_id=stream[1],pointList = streamPoints):
commitFlag = 1
'''
for p in streamPoints:
result = datamanager.getDataPoint(stream[0],stream[1],p[0],p[1])
if result is None:
commitFlag = 1
datamanager.addDataPoint(dev_id=stream[0],stream_id=stream[1],timestamp = p[0],datapoint=p[1])
newDataPointCounter+=1
#app.logger.debug("New Data Point Record: ",stream,p)
'''
if commitFlag:
datamanager.commitDB()
else:
print "\tNo new data"
endTimer = startTimer
print "Data Points Added:",newDataPointCounter
return newDataPointCounter
def updateDeviceList(self):
response_body = self.genericWebServiceCall("/DeviceCore","GET")
result = None
try:
if "Bad credentials" in response_body:
return None
except:
return None
self.deviceListInfo = xmlParse.parseDeviceListing(response_body) #gets only from Etherios
#local
#([connectID,lat,longit,group,connected,globID,disconnectTime ])
for i in self.deviceListInfo:
print i
try:
result = datamanager.getDeviceByID(i[0])
except:
pass
if result is None:
print "\tNEW Device","\t",i[1],i[3]
if i[0][0] == "0":
i[0].upper()
datamanager.addNewDevice(dev_connectware_id=i[0],dp_map_lat=i[1],dp_map_long=i[2],dp_connection_status=i[4],dp_global_ip=i[5],dp_last_disconnect_time=i[6])
#recordItem = models.device(dev_connectware_id=i[0],dp_map_lat=i[1],dp_map_long=i[2],dp_connection_status=i[4],dp_global_ip=i[5],dp_last_disconnect_time=i[6])
#db.session.add(recordItem)
else:
print "\tUPDATE Device","\t",result.dev_connectware_id,result.dp_connection_status
result.dev_connectware_id=i[0]
result.dp_map_lat=i[1]
result.dp_map_long=i[2]
result.dp_connection_status=i[4]
result.dp_global_ip=i[5]
result.dp_last_disconnect_time=i[6]
print "Committing Device List..."
datamanager.commitDB()
self.deviceListInfo = datamanager.getDeviceListFormatted()
return self.deviceListInfo
def updateLatestStreamValues(self):
result = None
response_body = self.genericWebServiceCall("/DataStream/","GET")
if "Bad credentials" in response_body:
return None
self.streamListInfo = xmlParse.parseStreamListingXML(response_body)
#(dev_id,stream_id,TS,datapoint)
for i in self.streamListInfo:
#result = models.latestDataStreamPoints.query.filter_by(stream_id=i[1],dev_id=i[0]).first()
result = datamanager.getStreamListByDeviceIDAndstream_id(i[0],i[1])
if result is None:
print "\tNEW RECORD","\t",i#[2],i[3]
datamanager.addNewStream(dev_id=i[0],stream_id=i[1],timestamp = i[2],datapoint=i[3])
#recordItem = models.latestDataStreamPoints(dev_id=i[0],stream_id=i[1],timestamp = i[2],datapoint=i[3])
#db.session.add(recordItem)
else:
print "\tUPDATE RECORD","\t",result.dev_id,result.stream_id,result.datapoint,str(time.strftime('%B %d, %Y %H:%M:%S', time.localtime((float(result.timestamp)/1000))))
result.timestamp = i[2]
result.datapoint=i[3]
datamanager.commitDB()
return self.streamListInfo
