def GetSensors(self, user, scpuser, scppasswd, source, dest):
pw = pwd.getpwnam(user)
uid = pw.pw_uid
os.setuid(uid)
print "setuid successful to", uid
try:
scptransfer(scpuser+'@'+source,dest,scppasswd)
except:
print "Could not connect to/get sensor info of client %s" % clientname
def subscribeSensor(self,client,output,module,sensorshort,sensorid):
"""
Subscribing to Sensors:
principally any subscrition is possible if the subscription string is suppported by the moons protocols
"""
self.prefix(module, "http://example.com/" + client +"/"+module+"#")
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % sensorid
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
log.msg("collectors client: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg("collectors client: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# if not present then get a file and upload it
log.msg("collectors client: No sensors registered so far - Getting data file from moon and uploading it using stream2db")
day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d')
for exten in ['bin','asc']:
destfile = os.path.join(destpath,'MartasFiles', sensorid+'_'+day+'.'+exten)
datafile = os.path.join('/srv/ws/', clientname, sensorid, sensorid+'_'+day+'.'+exten)
try:
log.msg("collectors client: Downloading data: %s" % datafile)
scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1])
stream = st.read(destfile)
log.msg("collectors client: Reading with MagPy... Found: %s datapoints" % str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = sensorshort
try:
stream.header['SensorRevision'] = sensorid[-4:]
except:
log.msg("collectors client: Could not extract revision number for %s" % sensorid)
pass
try:
stream.header['SensorSerialNum'] = sensorid.split('_')[-2]
except:
log.msg("collectors client: Could not extract serial number for %s" % sensorid)
pass
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db,stream)
except:
log.msg("collectors client: Could not upload data to the data base - subscription failed")
else:
log.msg("collectors client: Found sensor(s) in DB - subscribing to the highest revision number")
subscriptionstring = "%s:%s-value" % (module, sensorid)
print "collectors sensor client: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors client: Running for sensor", sensorid
subscriptionstring = "%s:%s-value" % (module, sensorid)
self.subscribe(subscriptionstring, self.onEvent)
def subscribeOw(self, client, output, module, owlist):
"""
Subscribing to all Onewire Instruments
"""
self.prefix(module, "http://example.com/" + client +"/"+module+"#")
if output == 'db':
# -------------------------------------------------------
# A. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in owlist:
subs = True
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[0]
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
log.msg("collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg("collectors owclient: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg("collectors owclient: No sensors registered so far - Getting file from moon and uploading it")
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d')
destfile = os.path.join(destpath,'MartasFiles', row[0]+'_'+day+'.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0], row[0]+'_'+day+'.bin')
try:
log.msg("collectors owclient: Downloading data: %s" % datafile)
scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1])
stream = st.read(destfile)
log.msg("collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db,stream)
log.msg("collectors owclient: Stream uploaded successfully")
except:
log.msg("collectors owclient: Could not upload data to the data base - subscription to %s failed" % row[0])
subs = False
else:
log.msg("collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number")
if subs:
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to DB): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to file): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
def subscribeSensor(self, client, output, module, sensorshort, sensorid):
"""
Subscribing to Sensors:
principally any subscrition is possible if the subscription string is suppported by the moons protocols
"""
self.prefix(module,
"http://example.com/" + client + "/" + module + "#")
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % sensorid
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
log.msg("collectors client: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg(
"collectors client: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# if not present then get a file and upload it
log.msg(
"collectors client: No sensors registered so far - Getting data file from moon and uploading it using stream2db"
)
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
for exten in ['bin', 'asc']:
destfile = os.path.join(destpath, 'MartasFiles',
sensorid + '_' + day + '.' + exten)
datafile = os.path.join('/srv/ws/', clientname, sensorid,
sensorid + '_' + day + '.' + exten)
try:
log.msg("collectors client: Downloading data: %s" %
datafile)
scptransfer(
sshcred[0] + '@' + clientip + ':' + datafile,
destfile, sshcred[1])
stream = st.read(destfile)
log.msg(
"collectors client: Reading with MagPy... Found: %s datapoints"
% str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = sensorshort
try:
stream.header['SensorRevision'] = sensorid[-4:]
except:
log.msg(
"collectors client: Could not extract revision number for %s"
% sensorid)
pass
try:
stream.header['SensorSerialNum'] = sensorid.split(
'_')[-2]
except:
log.msg(
"collectors client: Could not extract serial number for %s"
% sensorid)
pass
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db, stream)
except:
log.msg(
"collectors client: Could not upload data to the data base - subscription failed"
)
else:
log.msg(
"collectors client: Found sensor(s) in DB - subscribing to the highest revision number"
)
subscriptionstring = "%s:%s-value" % (module, sensorid)
print "collectors sensor client: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors client: Running for sensor", sensorid
subscriptionstring = "%s:%s-value" % (module, sensorid)
self.subscribe(subscriptionstring, self.onEvent)
def subscribeOw(self, client, output, module, owlist):
"""
Subscribing to all Onewire Instruments
"""
self.prefix(module,
"http://example.com/" + client + "/" + module + "#")
if output == 'db':
# -------------------------------------------------------
# A. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in owlist:
subs = True
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[
0]
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
log.msg(
"collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg(
"collectors owclient: Unable to fetch SENSOR data from DB"
)
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg(
"collectors owclient: No sensors registered so far - Getting file from moon and uploading it"
)
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
destfile = os.path.join(destpath, 'MartasFiles',
row[0] + '_' + day + '.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0],
row[0] + '_' + day + '.bin')
try:
log.msg("collectors owclient: Downloading data: %s" %
datafile)
scptransfer(
sshcred[0] + '@' + clientip + ':' + datafile,
destfile, sshcred[1])
stream = st.read(destfile)
log.msg(
"collectors owclient: Reading with MagPy... Found: %s datapoints"
% str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db, stream)
log.msg(
"collectors owclient: Stream uploaded successfully"
)
except:
log.msg(
"collectors owclient: Could not upload data to the data base - subscription to %s failed"
% row[0])
subs = False
else:
log.msg(
"collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number"
)
if subs:
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to DB): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to file): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
#p = Process(target=MI.GetSensors, args=('cobs',scpuser,scppasswd,source,destsensfile,))
#p.start()
#p.join()
#source = clientip+':'+owfile
#p = Process(target=MI.GetSensors, args=('cobs',scpuser,scppasswd,source,destowfile,))
#p.start()
#p.join()
# Please note: scp is not workings from root
# Therefore the following processes are performed as defaultuser (ideally as a subprocess)
uid=pwd.getpwnam(defaultuser)[2]
os.setuid(uid)
try:
print scpuser+'@'+clientip+':'+sensfile
scptransfer(scpuser+'@'+clientip+':'+sensfile,destsensfile,scppasswd)
except:
print "Could not connect to/get sensor info of client %s - aborting" % clientname
sys.exit()
print "Searching for onewire data from ", clientname
try:
scptransfer(scpuser+'@'+clientip+':'+owfile,destowfile,scppasswd)
except:
print "No one wire info available on client %s - proceeding" % clientname
pass
s,o = [],[]
with open(destsensfile,'rb') as f:
reader = csv.reader(f)
s = []
for line in reader:
def subscribeInst(self, db, cursor, client, mod, output):
print "Starting Subscription ...."
self.prefix(mod, "http://example.com/" + client +"/"+mod+"#")
if mod == 'cs':
sensshort = 'G82'
else:
sensshort = mod.upper()
if mod == 'ow':
if not len(o) > 0:
log.msg('collectors client: No OW sensors available')
else:
log.msg('Subscribing OneWire Sensors ...')
self.subscribeOw(o)
else:
self.subscribeSensor(sensshort)
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# -------------------------------------------------------
# 2. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in o:
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[0]
print sql
try:
# Execute the SQL command
cursor.execute(sql)
except:
log.msg("collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
print "SQL-Results:", results
except:
log.msg("collectors owclient: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg("collectors owclient: No sensors registered so far - Getting file from moon and uploading it")
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d')
destfile = os.path.join(destpath,'MoonsFiles', row[0]+'_'+day+'.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0], row[0]+'_'+day+'.bin')
try:
log.msg("collectors owclient: Downloading data: %s" % datafile)
scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1])
stream = st.read(destfile)
log.msg("collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
self.sensorgroup = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
stream2db(db,stream)
self.sensorid = stream.header['SensorID']
self.sensortype = row[1]
except:
log.msg("collectors owclient: Could not upload data to the data base - subscription failed")
elif len(results) == 1:
log.msg("collectors owclient: Found sensor in db - subscribing to its table")
self.sensorid = results[0][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
else:
log.msg("collectors owclient: Found several sensors in db - subscribing the highest revision number")
self.sensorid = results[-1][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (mod, row[0])
self.subscribe(subscriptionstring, self.onEvent)
def subscribeInst(self, db, cursor, client, mod, output):
print "Starting Subscription ...."
self.prefix(mod, "http://example.com/" + client + "/" + mod + "#")
if mod == 'cs':
sensshort = 'G82'
else:
sensshort = mod.upper()
if mod == 'ow':
if not len(o) > 0:
log.msg('collectors client: No OW sensors available')
else:
log.msg('Subscribing OneWire Sensors ...')
self.subscribeOw(o)
else:
self.subscribeSensor(sensshort)
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# -------------------------------------------------------
# 2. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in o:
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[
0]
print sql
try:
# Execute the SQL command
cursor.execute(sql)
except:
log.msg(
"collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
print "SQL-Results:", results
except:
log.msg(
"collectors owclient: Unable to fetch SENSOR data from DB"
)
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg(
"collectors owclient: No sensors registered so far - Getting file from moon and uploading it"
)
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
destfile = os.path.join(destpath, 'MoonsFiles',
row[0] + '_' + day + '.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0],
row[0] + '_' + day + '.bin')
try:
log.msg("collectors owclient: Downloading data: %s" %
datafile)
scptransfer(
sshcred[0] + '@' + clientip + ':' + datafile,
destfile, sshcred[1])
stream = st.read(destfile)
log.msg(
"collectors owclient: Reading with MagPy... Found: %s datapoints"
% str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
self.sensorgroup = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
stream2db(db, stream)
self.sensorid = stream.header['SensorID']
self.sensortype = row[1]
except:
log.msg(
"collectors owclient: Could not upload data to the data base - subscription failed"
)
elif len(results) == 1:
log.msg(
"collectors owclient: Found sensor in db - subscribing to its table"
)
self.sensorid = results[0][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
else:
log.msg(
"collectors owclient: Found several sensors in db - subscribing the highest revision number"
)
self.sensorid = results[-1][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (mod, row[0])
self.subscribe(subscriptionstring, self.onEvent)
print "MySQL server version:", row[0]
cursor.close ()
db.close ()
# ----------------------------------------------------------
# 3. connect to client and get sensor list as well as owlist
# ----------------------------------------------------------
print "Locating MARCOS directory ..."
destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
sensfile = os.path.join(martaspath,'sensors.txt')
owfile = os.path.join(martaspath,'owlist.csv')
destsensfile = os.path.join(destpath,'MoonsSensors',clientname+'_sensors.txt')
destowfile = os.path.join(destpath,'MoonsSensors',clientname+'_owlist.csv')
print "Getting sensor information from ", clientname
try:
scptransfer(scpuser+'@'+clientip+':'+sensfile,destsensfile,scppasswd)
except:
print "Could not connect to/get sensor info of client %s - aborting" % clientname
sys.exit()
print "Searching for onewire data from ", clientname
try:
scptransfer(scpuser+'@'+clientip+':'+owfile,destowfile,scppasswd)
except:
print "No one wire info available on client %s - proceeding" % clientname
pass
s,o = [],[]
with open(destsensfile,'rb') as f:
reader = csv.reader(f)
s = []
for line in reader:
