def collect_all(self, socket): for tbl in self.list_of_tables: self.log.debug("Collecting all from {}.{}".format(self.metric_id, tbl.name)) more_flag = True # collect most recent record data, more_flag = pakbus.collect_data(socket, self.pakbus_id, self.my_node_id, self.tabledef, tbl.name, CollectMode = 0x03) if data and len(data) > 0: self.last_collect['ts'] = datetime.datetime.now() # compute next record to collect self.last_collect['NextRecNbr'] = data[0]['BegRecNbr'] + data[0]['NbrOfRecs'] self.emit_all(tbl, data[0]['RecFrag']) self.update_local_db(self.last_collect['ts'], self.last_collect['NextRecNbr']) #if self.data_callback: #for rec in data[0]['RecFrag']: #self.data_callback(self, rec) if more_flag: self.collect_increment(socket)
def fetchdata(self, t_name = "Snow", lastrecs = 0): """ fetch the data form a table on the logger arguments: t_name: (string) name of the table lastrecs: (int) the last record recived returns: the data in an array """ self.ping() array = [] try: recs, more = pakbus.collect_data(self.link, self.l_id, self.c_id , self.tabledef , t_name,P1 = 1) except StandardError: print "Error: table " + t_name + " was not found." if t_name == "Status" or t_name == "Public": return recs[0]['RecFrag'] numRecs = recs[0]['RecFrag'][0]['RecNbr'] +1 if lastrecs != 0: numRecs -= (lastrecs + 1) #~ print numRecs more = 1 while numRecs != 0: recs, more = pakbus.collect_data(self.link, self.l_id, self.c_id , self.tabledef , t_name,P1 = numRecs) if len(recs) < 1 : return array for items in recs[0]['RecFrag']: array.append(items) temp = recs[0]['NbrOfRecs'] #~ print temp numRecs -= temp #~ print numRecs return array
def get_table_data (self, table_name, last_recived = 0): """ Function doc """ table = [] while True: data, is_more = pkb.collect_data(self.socket, self.logger_id, self.cpu_id, self.table_def, table_name, CollectMode = 0x04, P1 = last_recived) #~ print data try: for rec in (data[0]['RecFrag']): table.append(rec) last_recived += data[0]['NbrOfRecs'] except: break return table
def get_table_data (self, table_name, last_recived = 0, show = False): """ Function doc """ #~ print "GETTING DATA" table = [] self.open_socket() while True: if show: try: print "Last Record Recived: " + str(table[-1]['RecNbr']) except IndexError: pass try: #self.open_socket() data, is_more = pkb.collect_data(self.socket, self.logger_id, self.cpu_id, self.table_def, table_name, CollectMode = 0x04, P1 = last_recived) except KeyError: print last_recived continue except struct.error: print "struct.error, but this may be the last record so returnign data" return table #~ print data try: for rec in (data[0]['RecFrag']): table.append(rec) last_recived += data[0]['NbrOfRecs'] except: break #~ if is_more == 0: #~ break return table
def collect_increment(self, socket): for tbl in self.list_of_tables: self.log.debug("Collecting increment from {}.{} NextRecNbr = {}".format(self.metric_id, tbl.name, self.last_collect['NextRecNbr'])) more_flag = True while more_flag: # collect from where we left off data, more_flag = pakbus.collect_data(socket, self.pakbus_id, self.my_node_id, self.tabledef, tbl.name, CollectMode = 0x04, P1 = self.last_collect['NextRecNbr']) if data and len(data) > 0: self.last_collect['ts'] = datetime.datetime.now() # compute next record to collect self.last_collect['NextRecNbr'] = data[0]['BegRecNbr'] + data[0]['NbrOfRecs'] # possible loss of data, we should wait for emit all to complete before # committing this... but forget it for now. self.update_local_db(self.last_collect['ts'], self.last_collect['NextRecNbr']) self.emit_all(tbl, data[0]['RecFrag']) if more_flag: self.log.info("{}.{} has more records to fetch".format(self.metric_id, tbl.name)) else: self.log.info("{}.{} is current".format(self.metric_id, tbl.name)) #if self.data_callback: #for rec in data[0]['RecFrag']: #self.data_callback(tbl.dic_of_sensorTags, rec) else: self.log.warn("Collecting data but got no records.")
print "datalogger time: ", new_data[0][0]['RecFrag'][0]['TimeOfRec'][0] diff = time.time()-time.mktime(time.strptime("31 Dec 1989 22 59 57", "%d %b %Y %H %M %S")) print "error: ", diff - new_data[0][0]['RecFrag'][0]['TimeOfRec'][0] """ import time print time.time() print time.localtime() print time.strptime("1 Jan 1990", "%d %b %Y") print "1/1/1990: ", time.mktime(time.strptime("1 Jan 1990", "%d %b %Y")) diff = time.time() - time.mktime(time.strptime("1 Jan 1990", "%d %b %Y")) print "current time - 1/1/1990: ", diff # print "error: ", diff - data[0][0]['RecFrag'][0]['TimeOfRec'][0] data = pakbus.collect_data(s, NodeId, MyNodeId, tabledef, "Table", CollectMode=0x04, P1=67) print "timestamp: ", data[0][0]["RecFrag"][0]["TimeOfRec"][0] print time.mktime(time.strptime("12 Jun 2014 11 56 00", "%d %b %Y %H %M %S")) """ #oldData = pakbus.collect_data(s, NodeId, MyNodeId, tabledef, 'Table') #oldDataTime = oldData[0][0]['RecFrag'][0]['TimeOfRec'][0] start_time = 770901795 end_time = 770901885 start_time -= 15 while start_time < end_time: data = pakbus.collect_data(s, NodeId, MyNodeId, tabledef, 'Table',CollectMode = 0x07, P1 = start_time, P2 = start_time) start_time = start_time+15 #if data[0][0]['RecFrag'][0]['TimeOfRec'][0] != oldDataTime:
diff = time.time()-time.mktime(time.strptime("31 Dec 1989 22 59 57", "%d %b %Y %H %M %S")) print "error: ", diff - new_data[0][0]['RecFrag'][0]['TimeOfRec'][0] ''' import time print time.time() print time.localtime() print time.strptime("1 Jan 1990", "%d %b %Y") print "1/1/1990: ", time.mktime(time.strptime("1 Jan 1990", "%d %b %Y")) diff = time.time() - time.mktime(time.strptime("1 Jan 1990", "%d %b %Y")) print "current time - 1/1/1990: ", diff #print "error: ", diff - data[0][0]['RecFrag'][0]['TimeOfRec'][0] data = pakbus.collect_data(s, NodeId, MyNodeId, tabledef, 'Table', CollectMode=0x04, P1=67) print 'timestamp: ', data[0][0]['RecFrag'][0]['TimeOfRec'][0] print time.mktime(time.strptime("12 Jun 2014 11 56 00", "%d %b %Y %H %M %S")) ''' #oldData = pakbus.collect_data(s, NodeId, MyNodeId, tabledef, 'Table') #oldDataTime = oldData[0][0]['RecFrag'][0]['TimeOfRec'][0] start_time = 770901795 end_time = 770901885 start_time -= 15 while start_time < end_time: data = pakbus.collect_data(s, NodeId, MyNodeId, tabledef, 'Table',CollectMode = 0x07, P1 = start_time, P2 = start_time)
def collectData(self, addition_size, num_dataloggers): ''' Collect data from datalogger up to "addition_size" number of records ''' raise Exception ("depreciated") up_to_date = False datalogger_size = len(self.list_of_tables) start_record_numbers = [0]*datalogger_size all_start_records = [0]*num_dataloggers try: if os.stat("lastRecord.txt").st_size > 0: with open("lastRecord.txt", "r") as f: all_start_records = f.read().split('\n') start_record_numbers = all_start_records[self.number].split(',') else: # empty file start_record_numbers = [0]*datalogger_size except OSError: # no file start_record_numbers = [0]*datalogger_size end_record_numbers = [int(x) for x in start_record_numbers] #print 'datalogger_size: ', datalogger_size for i in range(datalogger_size): data = pakbus.collect_data(self.socket, self.pakbus_id, self.my_node_id, self.tabledef, self.list_of_tables[i].name, CollectMode = 0x05, P1 = 1) start_time = time.time() if (len(data[0]) == 0): # rarely happends # dont know the cause #print 'got here for some reason' break most_recent_record_number = data[0][0]['RecFrag'][0]['RecNbr'] data_to_add_to_database = [] current_table = self.list_of_tables[i] record_number = int(start_record_numbers[i]) for j in range(addition_size): data = pakbus.collect_data(self.socket, self.pakbus_id, self.my_node_id, self.tabledef, self.list_of_tables[i].name, \ CollectMode = 0x06, P1 = record_number, P2 = record_number+1) if (len(data[0]) == 0): # gets here because msg is an empty dictionary # only happens for CR800, as if fails connections sometimes # never happens for CR1000 # line 1527 in pakbus.py # got here because there is no data j -= 1 # dont add so dont count #print 'got here instead' break time_stamp = data[0][0]['RecFrag'][0]['TimeOfRec'][0] + 631166400 sensors = data[0][0]['RecFrag'][0]['Fields'] #print data for sensor in sensors.keys(): # to avoid problem with "NaN" try: long(sensors[sensor][0]) data_to_add_to_database += [(time_stamp, sensors[sensor][0], self.list_of_tables[i].dic_of_sensorTags[sensor].tag)] except: print 'a data point was NaN or invalid' # exclude data point # data_to_add_to_database += [(time_stamp, -1, self.list_of_tables[i].dic_of_sensorTags[sensor].tag)] data_to_add_to_database += [(time_stamp, record_number, self.list_of_tables[i].record_tag)] record_number += 1 if (record_number > most_recent_record_number): up_to_date = True #print 'got here' break else: up_to_date = False #print len(data_to_add_to_database) #print time.time()-start_time #start_time = time.time() #print data_to_add_to_database #return if (len(data_to_add_to_database) > 0): print client.multiplePut(data_to_add_to_database) + ' records added: ' + str(j+1) #print time.time()-start_time end_record_numbers[i] = record_number all_start_records[self.number] = str(end_record_numbers)[1:-1] all_end_records = "" for records in all_start_records: all_end_records = all_end_records + str(records) + '\n' all_end_records = all_end_records[0:-1] with open("lastRecord.txt", "w") as f: f.write(all_end_records) return up_to_date