def updateStreamListDataPoints(self,fromDate=0,limit=10): ''' May want to allow for getting more than 1000 points per stream, Etherios Limited by default... Adjust request size or ask for more if available if limit hit... This module however should not miss any datapoints as it retrieves data since latest point sampled Even though it may not bring all data in at once, calling multiple times will eventually get all data if it isnt coming in faster than the call rate Ex. if calling daily, would need to verify that no more than 1000 points/day per stream would typically be generated Requires an export rate faster than 1 point every ~1.5 minutes per stream to need multiple calls per day ''' '''May need to optimize insert method to use SQLAlchemy core method to speed up bulk inserts... potential to be 25 times faster.. 12 minute run time with maxed point counts and 15 streams''' endTimer=0 newDataPointCounter=0 commitFlag=0 for stream in self.streamListInfo: #for every data stream, get list of points if fromDate <= 0: lastPointTS = datamanager.getMostRecentTSDataPoint(stream[0],stream[1])+1 #add ms as to not query same data again else: lastPointTS = fromDate startTimer = time.time() #prevent oversampling Etherios if startTimer - endTimer < 1:#print "Fast Sample, Add Delay...",startTimer - endTimer print "Delay Added:",startTimer - endTimer time.sleep(1.2) else:pass#print "Sample Time Delta:", str(startTimer - endTimer) if lastPointTS: streamPoints = self.getDataStreamPoints(stream[0],stream[1],startTime=lastPointTS,limit=limit) else: print "Last data point not found!" streamPoints = self.getDataStreamPoints(stream[0],stream[1],limit=limit) if streamPoints is None: print "\tNo New Etherios data" #Stream exists, but no data points! Should this stream be deleted from Etherios???? #OR.....Zero results depending responses data elif len(streamPoints) == 0: #shouldnt happen print "\tNo New Etherios data" else: print "Processing ",len(streamPoints)," Data Points..." newDataPointCounter+=len(streamPoints) if datamanager.fastaddDataPoints(dev_id=stream[0],stream_id=stream[1],pointList = streamPoints): commitFlag = 1 ''' for p in streamPoints: result = datamanager.getDataPoint(stream[0],stream[1],p[0],p[1]) if result is None: commitFlag = 1 datamanager.addDataPoint(dev_id=stream[0],stream_id=stream[1],timestamp = p[0],datapoint=p[1]) newDataPointCounter+=1 #app.logger.debug("New Data Point Record: ",stream,p) ''' if commitFlag: datamanager.commitDB() else: print "\tNo new data" endTimer = startTimer print "Data Points Added:",newDataPointCounter return newDataPointCounter
def updateDeviceList(self): response_body = self.genericWebServiceCall("/DeviceCore","GET") result = None try: if "Bad credentials" in response_body: return None except: return None self.deviceListInfo = xmlParse.parseDeviceListing(response_body) #gets only from Etherios #local #([connectID,lat,longit,group,connected,globID,disconnectTime ]) for i in self.deviceListInfo: print i try: result = datamanager.getDeviceByID(i[0]) except: pass if result is None: print "\tNEW Device","\t",i[1],i[3] if i[0][0] == "0": i[0].upper() datamanager.addNewDevice(dev_connectware_id=i[0],dp_map_lat=i[1],dp_map_long=i[2],dp_connection_status=i[4],dp_global_ip=i[5],dp_last_disconnect_time=i[6]) #recordItem = models.device(dev_connectware_id=i[0],dp_map_lat=i[1],dp_map_long=i[2],dp_connection_status=i[4],dp_global_ip=i[5],dp_last_disconnect_time=i[6]) #db.session.add(recordItem) else: print "\tUPDATE Device","\t",result.dev_connectware_id,result.dp_connection_status result.dev_connectware_id=i[0] result.dp_map_lat=i[1] result.dp_map_long=i[2] result.dp_connection_status=i[4] result.dp_global_ip=i[5] result.dp_last_disconnect_time=i[6] print "Committing Device List..." datamanager.commitDB() self.deviceListInfo = datamanager.getDeviceListFormatted() return self.deviceListInfo
def updateLatestStreamValues(self): result = None response_body = self.genericWebServiceCall("/DataStream/","GET") if "Bad credentials" in response_body: return None self.streamListInfo = xmlParse.parseStreamListingXML(response_body) #(dev_id,stream_id,TS,datapoint) for i in self.streamListInfo: #result = models.latestDataStreamPoints.query.filter_by(stream_id=i[1],dev_id=i[0]).first() result = datamanager.getStreamListByDeviceIDAndstream_id(i[0],i[1]) if result is None: print "\tNEW RECORD","\t",i#[2],i[3] datamanager.addNewStream(dev_id=i[0],stream_id=i[1],timestamp = i[2],datapoint=i[3]) #recordItem = models.latestDataStreamPoints(dev_id=i[0],stream_id=i[1],timestamp = i[2],datapoint=i[3]) #db.session.add(recordItem) else: print "\tUPDATE RECORD","\t",result.dev_id,result.stream_id,result.datapoint,str(time.strftime('%B %d, %Y %H:%M:%S', time.localtime((float(result.timestamp)/1000)))) result.timestamp = i[2] result.datapoint=i[3] datamanager.commitDB() return self.streamListInfo