def GetSensors(self, user, scpuser, scppasswd, source, dest): pw = pwd.getpwnam(user) uid = pw.pw_uid os.setuid(uid) print "setuid successful to", uid try: scptransfer(scpuser+'@'+source,dest,scppasswd) except: print "Could not connect to/get sensor info of client %s" % clientname
def subscribeSensor(self,client,output,module,sensorshort,sensorid): """ Subscribing to Sensors: principally any subscrition is possible if the subscription string is suppported by the moons protocols """ self.prefix(module, "http://example.com/" + client +"/"+module+"#") if output == 'db': # ------------------------------------------------------- # 1. Get available Sensors - read sensors.txt # ------------------------------------------------------- # Try to find sensor in db: sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % sensorid try: # Execute the SQL command self.cursor.execute(sql) except: log.msg("collectors client: Unable to execute SENSOR sql") try: # Fetch all the rows in a list of lists. results = self.cursor.fetchall() except: log.msg("collectors client: Unable to fetch SENSOR data from DB") results = [] if len(results) < 1: # if not present then get a file and upload it log.msg("collectors client: No sensors registered so far - Getting data file from moon and uploading it using stream2db") day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d') for exten in ['bin','asc']: destfile = os.path.join(destpath,'MartasFiles', sensorid+'_'+day+'.'+exten) datafile = os.path.join('/srv/ws/', clientname, sensorid, sensorid+'_'+day+'.'+exten) try: log.msg("collectors client: Downloading data: %s" % datafile) scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1]) stream = st.read(destfile) log.msg("collectors client: Reading with MagPy... Found: %s datapoints" % str(len(stream))) stream.header['StationID'] = self.stationid stream.header['SensorModule'] = sensorshort try: stream.header['SensorRevision'] = sensorid[-4:] except: log.msg("collectors client: Could not extract revision number for %s" % sensorid) pass try: stream.header['SensorSerialNum'] = sensorid.split('_')[-2] except: log.msg("collectors client: Could not extract serial number for %s" % sensorid) pass if not len(stream.ndarray[0]) > 0: stream = stream.linestruct2ndarray() stream2db(self.db,stream) except: log.msg("collectors client: Could not upload data to the data base - subscription failed") else: log.msg("collectors client: Found sensor(s) in DB - subscribing to the highest revision number") subscriptionstring = "%s:%s-value" % (module, sensorid) print "collectors sensor client: Subscribing: ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent) elif output == 'file': for row in o: print "collectors client: Running for sensor", sensorid subscriptionstring = "%s:%s-value" % (module, sensorid) self.subscribe(subscriptionstring, self.onEvent)
def subscribeOw(self, client, output, module, owlist): """ Subscribing to all Onewire Instruments """ self.prefix(module, "http://example.com/" + client +"/"+module+"#") if output == 'db': # ------------------------------------------------------- # A. Create database input # ------------------------------------------------------- # ideal way: upload an already existing file from moon for each sensor # check client for existing file: for row in owlist: subs = True print "collectors owclient: Running for sensor", row[0] # Try to find sensor in db: sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[0] try: # Execute the SQL command self.cursor.execute(sql) except: log.msg("collectors owclient: Unable to execute SENSOR sql") try: # Fetch all the rows in a list of lists. results = self.cursor.fetchall() except: log.msg("collectors owclient: Unable to fetch SENSOR data from DB") results = [] if len(results) < 1: # Initialize e.g. ow table log.msg("collectors owclient: No sensors registered so far - Getting file from moon and uploading it") # if not present then get a file and upload it #destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0] day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d') destfile = os.path.join(destpath,'MartasFiles', row[0]+'_'+day+'.bin') datafile = os.path.join('/srv/ws/', clientname, row[0], row[0]+'_'+day+'.bin') try: log.msg("collectors owclient: Downloading data: %s" % datafile) scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1]) stream = st.read(destfile) log.msg("collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream))) stream.header['StationID'] = self.stationid stream.header['SensorModule'] = 'OW' stream.header['SensorType'] = row[1] if not row[2] == 'typus': stream.header['SensorGroup'] = row[2] if not row[3] == 'location': stream.header['DataLocationReference'] = row[3] if not row[4] == 'info': stream.header['SensorDescription'] = row[4] if not len(stream.ndarray[0]) > 0: stream = stream.linestruct2ndarray() stream2db(self.db,stream) log.msg("collectors owclient: Stream uploaded successfully") except: log.msg("collectors owclient: Could not upload data to the data base - subscription to %s failed" % row[0]) subs = False else: log.msg("collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number") if subs: subscriptionstring = "%s:%s-value" % (module, row[0]) print "collectors owclient: Subscribing (directing to DB): ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent) elif output == 'file': for row in o: print "collectors owclient: Running for sensor", row[0] subscriptionstring = "%s:%s-value" % (module, row[0]) print "collectors owclient: Subscribing (directing to file): ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent)
def subscribeSensor(self, client, output, module, sensorshort, sensorid): """ Subscribing to Sensors: principally any subscrition is possible if the subscription string is suppported by the moons protocols """ self.prefix(module, "http://example.com/" + client + "/" + module + "#") if output == 'db': # ------------------------------------------------------- # 1. Get available Sensors - read sensors.txt # ------------------------------------------------------- # Try to find sensor in db: sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % sensorid try: # Execute the SQL command self.cursor.execute(sql) except: log.msg("collectors client: Unable to execute SENSOR sql") try: # Fetch all the rows in a list of lists. results = self.cursor.fetchall() except: log.msg( "collectors client: Unable to fetch SENSOR data from DB") results = [] if len(results) < 1: # if not present then get a file and upload it log.msg( "collectors client: No sensors registered so far - Getting data file from moon and uploading it using stream2db" ) day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d') for exten in ['bin', 'asc']: destfile = os.path.join(destpath, 'MartasFiles', sensorid + '_' + day + '.' + exten) datafile = os.path.join('/srv/ws/', clientname, sensorid, sensorid + '_' + day + '.' + exten) try: log.msg("collectors client: Downloading data: %s" % datafile) scptransfer( sshcred[0] + '@' + clientip + ':' + datafile, destfile, sshcred[1]) stream = st.read(destfile) log.msg( "collectors client: Reading with MagPy... Found: %s datapoints" % str(len(stream))) stream.header['StationID'] = self.stationid stream.header['SensorModule'] = sensorshort try: stream.header['SensorRevision'] = sensorid[-4:] except: log.msg( "collectors client: Could not extract revision number for %s" % sensorid) pass try: stream.header['SensorSerialNum'] = sensorid.split( '_')[-2] except: log.msg( "collectors client: Could not extract serial number for %s" % sensorid) pass if not len(stream.ndarray[0]) > 0: stream = stream.linestruct2ndarray() stream2db(self.db, stream) except: log.msg( "collectors client: Could not upload data to the data base - subscription failed" ) else: log.msg( "collectors client: Found sensor(s) in DB - subscribing to the highest revision number" ) subscriptionstring = "%s:%s-value" % (module, sensorid) print "collectors sensor client: Subscribing: ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent) elif output == 'file': for row in o: print "collectors client: Running for sensor", sensorid subscriptionstring = "%s:%s-value" % (module, sensorid) self.subscribe(subscriptionstring, self.onEvent)
def subscribeOw(self, client, output, module, owlist): """ Subscribing to all Onewire Instruments """ self.prefix(module, "http://example.com/" + client + "/" + module + "#") if output == 'db': # ------------------------------------------------------- # A. Create database input # ------------------------------------------------------- # ideal way: upload an already existing file from moon for each sensor # check client for existing file: for row in owlist: subs = True print "collectors owclient: Running for sensor", row[0] # Try to find sensor in db: sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[ 0] try: # Execute the SQL command self.cursor.execute(sql) except: log.msg( "collectors owclient: Unable to execute SENSOR sql") try: # Fetch all the rows in a list of lists. results = self.cursor.fetchall() except: log.msg( "collectors owclient: Unable to fetch SENSOR data from DB" ) results = [] if len(results) < 1: # Initialize e.g. ow table log.msg( "collectors owclient: No sensors registered so far - Getting file from moon and uploading it" ) # if not present then get a file and upload it #destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0] day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d') destfile = os.path.join(destpath, 'MartasFiles', row[0] + '_' + day + '.bin') datafile = os.path.join('/srv/ws/', clientname, row[0], row[0] + '_' + day + '.bin') try: log.msg("collectors owclient: Downloading data: %s" % datafile) scptransfer( sshcred[0] + '@' + clientip + ':' + datafile, destfile, sshcred[1]) stream = st.read(destfile) log.msg( "collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream))) stream.header['StationID'] = self.stationid stream.header['SensorModule'] = 'OW' stream.header['SensorType'] = row[1] if not row[2] == 'typus': stream.header['SensorGroup'] = row[2] if not row[3] == 'location': stream.header['DataLocationReference'] = row[3] if not row[4] == 'info': stream.header['SensorDescription'] = row[4] if not len(stream.ndarray[0]) > 0: stream = stream.linestruct2ndarray() stream2db(self.db, stream) log.msg( "collectors owclient: Stream uploaded successfully" ) except: log.msg( "collectors owclient: Could not upload data to the data base - subscription to %s failed" % row[0]) subs = False else: log.msg( "collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number" ) if subs: subscriptionstring = "%s:%s-value" % (module, row[0]) print "collectors owclient: Subscribing (directing to DB): ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent) elif output == 'file': for row in o: print "collectors owclient: Running for sensor", row[0] subscriptionstring = "%s:%s-value" % (module, row[0]) print "collectors owclient: Subscribing (directing to file): ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent)
#p = Process(target=MI.GetSensors, args=('cobs',scpuser,scppasswd,source,destsensfile,)) #p.start() #p.join() #source = clientip+':'+owfile #p = Process(target=MI.GetSensors, args=('cobs',scpuser,scppasswd,source,destowfile,)) #p.start() #p.join() # Please note: scp is not workings from root # Therefore the following processes are performed as defaultuser (ideally as a subprocess) uid=pwd.getpwnam(defaultuser)[2] os.setuid(uid) try: print scpuser+'@'+clientip+':'+sensfile scptransfer(scpuser+'@'+clientip+':'+sensfile,destsensfile,scppasswd) except: print "Could not connect to/get sensor info of client %s - aborting" % clientname sys.exit() print "Searching for onewire data from ", clientname try: scptransfer(scpuser+'@'+clientip+':'+owfile,destowfile,scppasswd) except: print "No one wire info available on client %s - proceeding" % clientname pass s,o = [],[] with open(destsensfile,'rb') as f: reader = csv.reader(f) s = [] for line in reader:
def subscribeInst(self, db, cursor, client, mod, output): print "Starting Subscription ...." self.prefix(mod, "http://example.com/" + client +"/"+mod+"#") if mod == 'cs': sensshort = 'G82' else: sensshort = mod.upper() if mod == 'ow': if not len(o) > 0: log.msg('collectors client: No OW sensors available') else: log.msg('Subscribing OneWire Sensors ...') self.subscribeOw(o) else: self.subscribeSensor(sensshort) if output == 'db': # ------------------------------------------------------- # 1. Get available Sensors - read sensors.txt # ------------------------------------------------------- # ------------------------------------------------------- # 2. Create database input # ------------------------------------------------------- # ideal way: upload an already existing file from moon for each sensor # check client for existing file: for row in o: print "collectors owclient: Running for sensor", row[0] # Try to find sensor in db: sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[0] print sql try: # Execute the SQL command cursor.execute(sql) except: log.msg("collectors owclient: Unable to execute SENSOR sql") try: # Fetch all the rows in a list of lists. results = self.cursor.fetchall() print "SQL-Results:", results except: log.msg("collectors owclient: Unable to fetch SENSOR data from DB") results = [] if len(results) < 1: # Initialize e.g. ow table log.msg("collectors owclient: No sensors registered so far - Getting file from moon and uploading it") # if not present then get a file and upload it #destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0] day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d') destfile = os.path.join(destpath,'MoonsFiles', row[0]+'_'+day+'.bin') datafile = os.path.join('/srv/ws/', clientname, row[0], row[0]+'_'+day+'.bin') try: log.msg("collectors owclient: Downloading data: %s" % datafile) scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1]) stream = st.read(destfile) log.msg("collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream))) stream.header['StationID'] = self.stationid stream.header['SensorModule'] = 'OW' stream.header['SensorType'] = row[1] if not row[2] == 'typus': stream.header['SensorGroup'] = row[2] self.sensorgroup = row[2] if not row[3] == 'location': stream.header['DataLocationReference'] = row[3] if not row[4] == 'info': stream.header['SensorDescription'] = row[4] stream2db(db,stream) self.sensorid = stream.header['SensorID'] self.sensortype = row[1] except: log.msg("collectors owclient: Could not upload data to the data base - subscription failed") elif len(results) == 1: log.msg("collectors owclient: Found sensor in db - subscribing to its table") self.sensorid = results[0][0] self.sensortype = row[1] if not row[2] == 'typus': self.sensorgroup = row[2] else: log.msg("collectors owclient: Found several sensors in db - subscribing the highest revision number") self.sensorid = results[-1][0] self.sensortype = row[1] if not row[2] == 'typus': self.sensorgroup = row[2] subscriptionstring = "%s:%s-value" % (module, row[0]) print "collectors owclient: Subscribing: ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent) elif output == 'file': for row in o: print "collectors owclient: Running for sensor", row[0] subscriptionstring = "%s:%s-value" % (mod, row[0]) self.subscribe(subscriptionstring, self.onEvent)
def subscribeInst(self, db, cursor, client, mod, output): print "Starting Subscription ...." self.prefix(mod, "http://example.com/" + client + "/" + mod + "#") if mod == 'cs': sensshort = 'G82' else: sensshort = mod.upper() if mod == 'ow': if not len(o) > 0: log.msg('collectors client: No OW sensors available') else: log.msg('Subscribing OneWire Sensors ...') self.subscribeOw(o) else: self.subscribeSensor(sensshort) if output == 'db': # ------------------------------------------------------- # 1. Get available Sensors - read sensors.txt # ------------------------------------------------------- # ------------------------------------------------------- # 2. Create database input # ------------------------------------------------------- # ideal way: upload an already existing file from moon for each sensor # check client for existing file: for row in o: print "collectors owclient: Running for sensor", row[0] # Try to find sensor in db: sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[ 0] print sql try: # Execute the SQL command cursor.execute(sql) except: log.msg( "collectors owclient: Unable to execute SENSOR sql") try: # Fetch all the rows in a list of lists. results = self.cursor.fetchall() print "SQL-Results:", results except: log.msg( "collectors owclient: Unable to fetch SENSOR data from DB" ) results = [] if len(results) < 1: # Initialize e.g. ow table log.msg( "collectors owclient: No sensors registered so far - Getting file from moon and uploading it" ) # if not present then get a file and upload it #destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0] day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d') destfile = os.path.join(destpath, 'MoonsFiles', row[0] + '_' + day + '.bin') datafile = os.path.join('/srv/ws/', clientname, row[0], row[0] + '_' + day + '.bin') try: log.msg("collectors owclient: Downloading data: %s" % datafile) scptransfer( sshcred[0] + '@' + clientip + ':' + datafile, destfile, sshcred[1]) stream = st.read(destfile) log.msg( "collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream))) stream.header['StationID'] = self.stationid stream.header['SensorModule'] = 'OW' stream.header['SensorType'] = row[1] if not row[2] == 'typus': stream.header['SensorGroup'] = row[2] self.sensorgroup = row[2] if not row[3] == 'location': stream.header['DataLocationReference'] = row[3] if not row[4] == 'info': stream.header['SensorDescription'] = row[4] stream2db(db, stream) self.sensorid = stream.header['SensorID'] self.sensortype = row[1] except: log.msg( "collectors owclient: Could not upload data to the data base - subscription failed" ) elif len(results) == 1: log.msg( "collectors owclient: Found sensor in db - subscribing to its table" ) self.sensorid = results[0][0] self.sensortype = row[1] if not row[2] == 'typus': self.sensorgroup = row[2] else: log.msg( "collectors owclient: Found several sensors in db - subscribing the highest revision number" ) self.sensorid = results[-1][0] self.sensortype = row[1] if not row[2] == 'typus': self.sensorgroup = row[2] subscriptionstring = "%s:%s-value" % (module, row[0]) print "collectors owclient: Subscribing: ", subscriptionstring self.subscribe(subscriptionstring, self.onEvent) elif output == 'file': for row in o: print "collectors owclient: Running for sensor", row[0] subscriptionstring = "%s:%s-value" % (mod, row[0]) self.subscribe(subscriptionstring, self.onEvent)
print "MySQL server version:", row[0] cursor.close () db.close () # ---------------------------------------------------------- # 3. connect to client and get sensor list as well as owlist # ---------------------------------------------------------- print "Locating MARCOS directory ..." destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0] sensfile = os.path.join(martaspath,'sensors.txt') owfile = os.path.join(martaspath,'owlist.csv') destsensfile = os.path.join(destpath,'MoonsSensors',clientname+'_sensors.txt') destowfile = os.path.join(destpath,'MoonsSensors',clientname+'_owlist.csv') print "Getting sensor information from ", clientname try: scptransfer(scpuser+'@'+clientip+':'+sensfile,destsensfile,scppasswd) except: print "Could not connect to/get sensor info of client %s - aborting" % clientname sys.exit() print "Searching for onewire data from ", clientname try: scptransfer(scpuser+'@'+clientip+':'+owfile,destowfile,scppasswd) except: print "No one wire info available on client %s - proceeding" % clientname pass s,o = [],[] with open(destsensfile,'rb') as f: reader = csv.reader(f) s = [] for line in reader: