def subscribeOw(self, client, output, module, owlist):
"""
Subscribing to all Onewire Instruments
"""
self.prefix(module, "http://example.com/" + client +"/"+module+"#")
if output == 'db':
# -------------------------------------------------------
# A. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in owlist:
subs = True
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[0]
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
log.msg("collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg("collectors owclient: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg("collectors owclient: No sensors registered so far - Getting file from moon and uploading it")
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d')
destfile = os.path.join(destpath,'MartasFiles', row[0]+'_'+day+'.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0], row[0]+'_'+day+'.bin')
try:
log.msg("collectors owclient: Downloading data: %s" % datafile)
scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1])
stream = st.read(destfile)
log.msg("collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db,stream)
log.msg("collectors owclient: Stream uploaded successfully")
except:
log.msg("collectors owclient: Could not upload data to the data base - subscription to %s failed" % row[0])
subs = False
else:
log.msg("collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number")
if subs:
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to DB): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to file): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
def subscribeSensor(self,client,output,module,sensorshort,sensorid):
"""
Subscribing to Sensors:
principally any subscrition is possible if the subscription string is suppported by the moons protocols
"""
self.prefix(module, "http://example.com/" + client +"/"+module+"#")
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % sensorid
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
log.msg("collectors client: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg("collectors client: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# if not present then get a file and upload it
log.msg("collectors client: No sensors registered so far - Getting data file from moon and uploading it using stream2db")
day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d')
for exten in ['bin','asc']:
destfile = os.path.join(destpath,'MartasFiles', sensorid+'_'+day+'.'+exten)
datafile = os.path.join('/srv/ws/', clientname, sensorid, sensorid+'_'+day+'.'+exten)
try:
log.msg("collectors client: Downloading data: %s" % datafile)
scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1])
stream = st.read(destfile)
log.msg("collectors client: Reading with MagPy... Found: %s datapoints" % str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = sensorshort
try:
stream.header['SensorRevision'] = sensorid[-4:]
except:
log.msg("collectors client: Could not extract revision number for %s" % sensorid)
pass
try:
stream.header['SensorSerialNum'] = sensorid.split('_')[-2]
except:
log.msg("collectors client: Could not extract serial number for %s" % sensorid)
pass
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db,stream)
except:
log.msg("collectors client: Could not upload data to the data base - subscription failed")
else:
log.msg("collectors client: Found sensor(s) in DB - subscribing to the highest revision number")
subscriptionstring = "%s:%s-value" % (module, sensorid)
print "collectors sensor client: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors client: Running for sensor", sensorid
subscriptionstring = "%s:%s-value" % (module, sensorid)
self.subscribe(subscriptionstring, self.onEvent)
def subscribeSensor(self, client, output, module, sensorshort, sensorid):
"""
Subscribing to Sensors:
principally any subscrition is possible if the subscription string is suppported by the moons protocols
"""
self.prefix(module,
"http://example.com/" + client + "/" + module + "#")
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % sensorid
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
print("collectors client: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
print("collectors client: Unable to fetch SENSOR data from DB")
results = []
# checking existing data table
try:
self.cursor.execute("SHOW TABLES LIKE '{}%'".format(sensorid))
secresults = self.cursor.fetchall()
except:
secresults = []
if len(results) < 1 and len(secresults) < 1:
# Initialize e.g. ow table
print("collectors client: No sensors registered so far ...")
if len(secresults) < 1:
# if not present then get a file and upload it
print(
"collectors client: No sensors registered so far - Getting data file from martas and uploading it using writeDB"
)
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
for exten in ['bin', 'asc']:
destfile = os.path.join(destpath, 'MartasFiles',
sensorid + '_' + day + '.' + exten)
datafile = os.path.join('/srv/ws/', clientname, sensorid,
sensorid + '_' + day + '.' + exten)
print("collectors client: Downloading data: {}".format(
datafile))
scptransfer(sshcred[0] + '@' + clientip + ':' + datafile,
destfile,
sshcred[1],
timeout=120)
print(
"collectors client: copying evetually existing data file to local directory {}"
.format(destfile))
try:
stream = st.read(destfile)
print(
"collectors client: Reading with MagPy... Found: {} datapoints"
.format(stream.length()[0]))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = sensorshort
try:
stream.header['SensorRevision'] = sensorid[-4:]
except:
print(
"collectors client: Could not extract revision number for %s"
% sensorid)
pass
try:
stream.header['SensorSerialNum'] = sensorid.split(
'_')[-2]
except:
print(
"collectors client: Could not extract serial number for %s"
% sensorid)
pass
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db, stream)
except:
print(
"collectors client: Could not upload data to the data base - subscription failed"
)
print(
" : possible reason: no ssh key established so far?"
)
else:
print(
"collectors client: Found sensor(s) in DB - subscribing to the highest revision number"
)
subscriptionstring = "%s:%s-value" % (module, sensorid)
print("collectors sensor client: Subscribing: ",
subscriptionstring)
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print("collectors client: Running for sensor", sensorid)
subscriptionstring = "%s:%s-value" % (module, sensorid)
self.subscribe(subscriptionstring, self.onEvent)
#!/bin/env/python
from magpy.stream import read
import magpy.mpplot as mp
from magpy.database import *
import magpy.opt.cred as mpcred
dbpasswd = mpcred.lc('cobsdb', 'passwd')
db = mysql.connect(host="138.22.188.195",
user="******",
passwd=dbpasswd,
db="cobsdb")
vario1 = read('/srv/archive/WIC/LEMI036_1_0002/raw/*2016-02-28.bin')
vario2 = read('/srv/archive/WIC/LEMI025_22_0002/raw/*2016-02-28.bin')
print vario1.length(), vario2.length()
mp.plotStreams([vario1, vario2], [['z'], ['z']])
def main(argv):
inputpath = ''
outputpath = ''
starttime = ''
endtime = ''
format_type = ''
coverage = None
metainfo = None
try:
opts, args = getopt.getopt(argv, "hr:w:s:e:f:m:c:", [
"read=", "write=", "sdate=", "edate=", "format=", "metainfo=",
"coverage="
])
except getopt.GetoptError:
print(
'convert.py -r <path2read> -w <path2write> -s <startdate> -e <enddate> -f <outputformat> -m <metainfo> -c <coverage>'
)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('Usage:')
print('------------')
print(
'convert.py -r <path2read> -w <path2write> -s <startdate> -e <enddate> -f <outputformat> -m <metainfo> -c <coverage>'
)
print('')
print('Description:')
print('------------')
print(
'Python program to convert bewteen data formats based on MagPy.'
)
print(
'Typical applications are the conversion of binary data formats'
)
print('to readable ASCII data sets or the conversion.')
print('')
print('Options:')
print('------------')
print(
'-r: specify file to read data from e.g. -r "/my/path/myfile.bin"'
)
print(
'-w: path to write data to: default is inputpath plus subfolder out'
)
print('-s: startdate: year-month-day e.g. 2018-01-23')
print('-e: enddate: e.g. 2018-01-26')
print(
'-c: coverage: one of "day" (DEFAULT),"month","year","all" ')
print('-f: supported output format:')
print(' -- PYASCII: MagPy basic ASCII --- (DEFAULT)')
print(' -- IAGA: IAGA 2002 text format')
print(' -- IMAGCDF: Intermagnet CDF Format')
print(' -- IAF: Intermagnet archive Format')
print(' -- WDC: World Data Centre format')
print(' -- IMF: Intermagnet Format')
print(' -- PYSTR: MagPy full ascii')
print(' -- PYCDF: MagPy CDF variant')
print(' -- DKA: K value format Intermagnet')
print(' -- DIDD: Output format from MinGeo DIDD')
print(' -- BLV: Baseline format Intermagnet')
print(' -- IYFV: Yearly mean format Intermagnet')
print(' -- JSON: JavaScript Object Notation')
print(' -- LATEX: LateX data')
print('-m: additional meta information:')
print(
' a string like "DataStandardLevel:Partial,StationIAGACode:WIC"'
)
print('')
print('Examples:')
print('------------')
print('Single file: mpconvert.py -r /my/path/myfile.bin')
print('Multiple files: mpconvert.py -r "/my/path/*.bin"')
print(
'Time range: mpconvert.py -r "/my/path/*" -s 2014-01-01 -e 2014-01-15'
)
print('Save as IAGA: mpconvert.py -r /my/path/myfile.bin -f IAGA')
print(
'IAGA to IMAGCDF: mpconvert.py -r /iagaseconds/201801* -f IMAGCDF -c month'
)
print(
'Define outputpath: mpconvert.py -r /my/input/path/myfile.bin -w /my/output/path/'
)
print('')
print('!!!! please note: !!!!')
print(
'!!!! using wildcards in filenames (asterix) requires quotes !!!!'
)
sys.exit()
elif opt in ("-r", "--read"):
inputpath = arg
elif opt in ("-w", "--write"):
outputpath = arg
elif opt in ("-s", "--sdate"):
starttime = arg
elif opt in ("-e", "--edate"):
endtime = arg
elif opt in ("-f", "--format"):
format_type = arg
elif opt in ("-m", "--metainfo"):
metainfo = arg
elif opt in ("-c", "--coverage"):
coverage = arg
if not inputpath:
print("mpconvert.py: no input data specified - aborting")
sys.exit()
if outputpath == '':
outputpath = inputpath
wildcard = False
try:
if '*' in os.path.split(outputpath)[1]:
wildcard = True
except:
pass
if os.path.isfile(outputpath) or outputpath.endswith('*') or wildcard:
outputpath = os.path.split(outputpath)[0]
outputpath = os.path.join(outputpath, 'out')
outputpath = os.path.abspath(outputpath)
if os.path.isfile(outputpath):
outputpath = os.path.split(outputpath)[0]
else:
# Test whether wpath is already a dirctory
if not os.path.isdir(outputpath):
if not os.path.exists(outputpath):
print(
"mpconvert.py: Directory {} not yet existing. Trying to create it..."
.format(outputpath))
try:
os.makedirs(outputpath)
print(" ... OK")
except:
print(" ... failed! - check permissions")
sys.exit()
else:
print('mpconvert.py: Error: cannot interpret path to write to')
sys.exit()
if inputpath == '':
print('mpconvert.py: Specify a file/path by the -r option:')
print('mpconvert.py: convert.py -r <path2read>')
print('mpconvert.py: -- check convert.py -h for more options')
sys.exit()
if format_type == '':
format_type = 'PYASCII'
else:
if not format_type in PYMAG_SUPPORTED_FORMATS:
print(
"mpconvert.py: Error: Unkown format! Choose one of the following output formats:"
)
for key in PYMAG_SUPPORTED_FORMATS:
val = PYMAG_SUPPORTED_FORMATS[key]
if 'w' in val[0] and len(val) > 1:
print(" -- {}: {}".format(key, val[1]))
sys.exit()
if not coverage in ['month', 'year', 'all']:
coverage = None
print("mpconvert.py: Starting conversion to {}:".format(format_type))
print(" loading data from {}".format(inputpath))
print(" and writing it to {}".format(outputpath))
print("Reading data ...")
if starttime != '' and endtime != '':
data = read(inputpath, starttime=starttime, endtime=endtime)
elif starttime != '':
data = read(inputpath, starttime=starttime)
elif endtime != '':
data = read(inputpath, endtime=endtime)
else:
data = read(inputpath)
num = len(data)
if num == 0:
print("mpconvert.py: Error: No readable data found")
print("mpconvert.py: Found {} timesteps ...".format(data.length()[0]))
if metainfo:
print("mpconvert.py: Adding/Replacing provided meta information")
metalist = metainfo.split(',')
for pair in metalist:
metadat = pair.split(':')
if len(metadat) == 2:
key = metadat[0]
value = metadat[1]
print(" -> {} = {}".format(key, value))
data.header[key] = value
print("mpconvert.py: Writing data to {} now...".format(outputpath))
# Check whether wpath is really a path
data.write(outputpath, format_type=format_type, coverage=coverage)
print("conversion finished!")
def subscribeOw(self, client, output, module, owlist):
"""
Subscribing to all Onewire Instruments
"""
self.prefix(module,
"http://example.com/" + client + "/" + module + "#")
if output == 'db':
# -------------------------------------------------------
# A. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in owlist:
subs = True
print("collectors owclient: Running for sensor", row[0])
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[
0]
try:
# Execute the SQL command
self.cursor.execute(sql)
except:
print("collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
print(
"collectors owclient: Unable to fetch SENSOR data from DB"
)
results = []
# checking existing data table
try:
self.cursor.execute(
"SHOW TABLES LIKE '{}%'".format(sensorid))
secresults = self.cursor.fetchall()
except:
secresults = []
if len(results) < 1 and len(secresults) < 1:
# Initialize e.g. ow table
print(
"collectors owclient: No sensors registered so far ..."
)
if len(secresults) < 1:
# Initialize e.g. ow table
print(
"collectors owclient: No data table existing so far - Getting file from moon and uploading it"
)
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
destfile = os.path.join(destpath, 'MartasFiles',
row[0] + '_' + day + '.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0],
row[0] + '_' + day + '.bin')
try:
print("collectors owclient: Downloading data: %s" %
datafile)
scptransfer(
sshcred[0] + '@' + clientip + ':' + datafile,
destfile, sshcred[1])
stream = st.read(destfile)
print(
"collectors owclient: Reading with MagPy... Found: {} datapoints"
.format(stream.length()[0]))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
if not len(stream.ndarray[0]) > 0:
stream = stream.linestruct2ndarray()
stream2db(self.db, stream)
print(
"collectors owclient: Stream uploaded successfully"
)
except:
print(
"collectors owclient: Could not upload data to the data base - subscription to %s failed"
% row[0])
subs = False
else:
print(
"collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number"
)
if subs:
subscriptionstring = "%s:%s-value" % (module, row[0])
print(
"collectors owclient: Subscribing (directing to DB): ",
subscriptionstring)
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print("collectors owclient: Running for sensor", row[0])
subscriptionstring = "%s:%s-value" % (module, row[0])
print("collectors owclient: Subscribing (directing to file): ",
subscriptionstring)
self.subscribe(subscriptionstring, self.onEvent)
def subscribeInst(self, db, cursor, client, mod, output):
print "Starting Subscription ...."
self.prefix(mod, "http://example.com/" + client +"/"+mod+"#")
if mod == 'cs':
sensshort = 'G82'
else:
sensshort = mod.upper()
if mod == 'ow':
if not len(o) > 0:
log.msg('collectors client: No OW sensors available')
else:
log.msg('Subscribing OneWire Sensors ...')
self.subscribeOw(o)
else:
self.subscribeSensor(sensshort)
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# -------------------------------------------------------
# 2. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in o:
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[0]
print sql
try:
# Execute the SQL command
cursor.execute(sql)
except:
log.msg("collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
print "SQL-Results:", results
except:
log.msg("collectors owclient: Unable to fetch SENSOR data from DB")
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg("collectors owclient: No sensors registered so far - Getting file from moon and uploading it")
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(),'%Y-%m-%d')
destfile = os.path.join(destpath,'MoonsFiles', row[0]+'_'+day+'.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0], row[0]+'_'+day+'.bin')
try:
log.msg("collectors owclient: Downloading data: %s" % datafile)
scptransfer(sshcred[0]+'@'+clientip+':'+datafile,destfile,sshcred[1])
stream = st.read(destfile)
log.msg("collectors owclient: Reading with MagPy... Found: %s datapoints" % str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
self.sensorgroup = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
stream2db(db,stream)
self.sensorid = stream.header['SensorID']
self.sensortype = row[1]
except:
log.msg("collectors owclient: Could not upload data to the data base - subscription failed")
elif len(results) == 1:
log.msg("collectors owclient: Found sensor in db - subscribing to its table")
self.sensorid = results[0][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
else:
log.msg("collectors owclient: Found several sensors in db - subscribing the highest revision number")
self.sensorid = results[-1][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (mod, row[0])
self.subscribe(subscriptionstring, self.onEvent)
deltaF=-0.258,
diid='A2_WIC.txt',
pier='A2',
expD=4.0,
expI=64.0,
starttime='2015-06-01',
endtime='2016-02-01',
db=db,
dbadd=True)
print absresult.length()
absresult.write('/srv/archive/WIC/DI/data',
coverage='all',
filenamebegins="BLV_FGE_S0252_0001_GSM90_14245_0002_A2_oc_min",
format_type="PYSTR",
mode='replace')
data1 = read(
'/srv/archive/WIC/DI/data/BLV_FGE_S0252_0001_GSM90_14245_0002_A2_oc_min.txt'
)
mp.plot(data1, ['dx', 'dy', 'dz'], symbollist=['o', 'o', 'o'])
data2 = read(
'/srv/archive/WIC/DI/data/BLV_FGE_S0252_0001_GSM90_14245_0002_A2.txt')
sub = subtractStreams(data1, data2)
mp.plot(sub, ['dx', 'dy', 'dz'], symbollist=['o', 'o', 'o'])
#writeDB(db,absresult,'BLV_FGE_S0252_0001_GSM90_14245_0002_A2')
def subscribeInst(self, db, cursor, client, mod, output):
print "Starting Subscription ...."
self.prefix(mod, "http://example.com/" + client + "/" + mod + "#")
if mod == 'cs':
sensshort = 'G82'
else:
sensshort = mod.upper()
if mod == 'ow':
if not len(o) > 0:
log.msg('collectors client: No OW sensors available')
else:
log.msg('Subscribing OneWire Sensors ...')
self.subscribeOw(o)
else:
self.subscribeSensor(sensshort)
if output == 'db':
# -------------------------------------------------------
# 1. Get available Sensors - read sensors.txt
# -------------------------------------------------------
# -------------------------------------------------------
# 2. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in o:
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[
0]
print sql
try:
# Execute the SQL command
cursor.execute(sql)
except:
log.msg(
"collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
print "SQL-Results:", results
except:
log.msg(
"collectors owclient: Unable to fetch SENSOR data from DB"
)
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg(
"collectors owclient: No sensors registered so far - Getting file from moon and uploading it"
)
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
destfile = os.path.join(destpath, 'MoonsFiles',
row[0] + '_' + day + '.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0],
row[0] + '_' + day + '.bin')
try:
log.msg("collectors owclient: Downloading data: %s" %
datafile)
scptransfer(
sshcred[0] + '@' + clientip + ':' + datafile,
destfile, sshcred[1])
stream = st.read(destfile)
log.msg(
"collectors owclient: Reading with MagPy... Found: %s datapoints"
% str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
self.sensorgroup = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
stream2db(db, stream)
self.sensorid = stream.header['SensorID']
self.sensortype = row[1]
except:
log.msg(
"collectors owclient: Could not upload data to the data base - subscription failed"
)
elif len(results) == 1:
log.msg(
"collectors owclient: Found sensor in db - subscribing to its table"
)
self.sensorid = results[0][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
else:
log.msg(
"collectors owclient: Found several sensors in db - subscribing the highest revision number"
)
self.sensorid = results[-1][0]
self.sensortype = row[1]
if not row[2] == 'typus':
self.sensorgroup = row[2]
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing: ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (mod, row[0])
self.subscribe(subscriptionstring, self.onEvent)
def subscribeOw(self, owlist):
"""
Subscribing to all Onewire Instruments
"""
if output == 'db':
# -------------------------------------------------------
# A. Create database input
# -------------------------------------------------------
# ideal way: upload an already existing file from moon for each sensor
# check client for existing file:
for row in owlist:
print "collectors owclient: Running for sensor", row[0]
# Try to find sensor in db:
sql = "SELECT SensorID FROM SENSORS WHERE SensorID LIKE '%s%%'" % row[
0]
try:
# Execute the SQL command
cursor.execute(sql)
except:
log.msg(
"collectors owclient: Unable to execute SENSOR sql")
try:
# Fetch all the rows in a list of lists.
results = self.cursor.fetchall()
except:
log.msg(
"collectors owclient: Unable to fetch SENSOR data from DB"
)
results = []
if len(results) < 1:
# Initialize e.g. ow table
log.msg(
"collectors owclient: No sensors registered so far - Getting file from moon and uploading it"
)
# if not present then get a file and upload it
#destpath = [path for path, dirs, files in os.walk("/home") if path.endswith('MARCOS')][0]
day = datetime.strftime(datetime.utcnow(), '%Y-%m-%d')
destfile = os.path.join(destpath, 'MoonsFiles',
row[0] + '_' + day + '.bin')
datafile = os.path.join('/srv/ws/', clientname, row[0],
row[0] + '_' + day + '.bin')
try:
log.msg("collectors owclient: Downloading data: %s" %
datafile)
scptransfer(
sshcred[0] + '@' + clientip + ':' + datafile,
destfile, sshcred[1])
stream = st.read(destfile)
log.msg(
"collectors owclient: Reading with MagPy... Found: %s datapoints"
% str(len(stream)))
stream.header['StationID'] = self.stationid
stream.header['SensorModule'] = 'OW'
stream.header['SensorType'] = row[1]
if not row[2] == 'typus':
stream.header['SensorGroup'] = row[2]
if not row[3] == 'location':
stream.header['DataLocationReference'] = row[3]
if not row[4] == 'info':
stream.header['SensorDescription'] = row[4]
stream2db(db, stream)
except:
log.msg(
"collectors owclient: Could not upload data to the data base - subscription failed"
)
else:
log.msg(
"collectors owclient: Found sensor(s) in DB - subscribing to the highest revision number"
)
subscriptionstring = "%s:%s-value" % (module, row[0])
print "collectors owclient: Subscribing (directing to DB): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
elif output == 'file':
for row in o:
print "collectors owclient: Running for sensor", row[0]
subscriptionstring = "%s:%s-value" % (mod, row[0])
print "collectors owclient: Subscribing (directing to file): ", subscriptionstring
self.subscribe(subscriptionstring, self.onEvent)
def process_ACE(datum, ace_P, ace_types, merge_variables, skipcompression=False, localpath='', debug=False):
"""
Processes new data and adds it to old stream.
INPUT:
ace_P: (str) String describing data type: '1m' or '5m'
ace_types : (list) List of type for type, e.g. ["swepam" and "mag"]
marge_variables: (list) Variables to merge from type 2 into 1, e.g.:
['x','y','z','f','t1','t2']
"""
print("Processing {} data for {} ...".format(ace_P,datum))
newday = False
# Read current data
if debug:
print (" Accessing {}".format(os.path.join(localpath,'raw','{}_ace_{}_{}.txt'.format(datum, ace_types[0], ace_P))))
ace_stream1 = read(os.path.join(localpath,'raw','{}_ace_{}_{}.txt'.format(datum, ace_types[0], ace_P)))
ace_stream2 = read(os.path.join(localpath,'raw','{}_ace_{}_{}.txt'.format(datum, ace_types[1], ace_P)))
if not ace_stream1.length()[0] > 0 and not ace_stream2.length()[0] > 0:
print (" No ACE data found - aborting")
else:
print ("{}: got {} datapoints".format(ace_types[0],ace_stream1.length()[0]))
print ("{}: got {} datapoints".format(ace_types[1],ace_stream2.length()[0]))
lastval = num2date(ace_stream1.ndarray[0][-1])
today = datetime.strftime(lastval, "%Y-%m-%d")
yesterday = datetime.strftime(lastval-timedelta(days=1), "%Y-%m-%d")
ace_file = os.path.join(localpath,'collected','ace_%s_%s.cdf' % (ace_P, today))
ace_lastfile = os.path.join(localpath,'collected','ace_%s_%s.cdf' % (ace_P, yesterday))
lastfile = True
if os.path.exists(ace_file):
try:
ace_last = read(ace_file)
except:
lastfile = False
else:
try:
ace_last = read(ace_lastfile)
newday = True
except:
lastfile = False
if lastfile:
if len(ace_last.ndarray[KEYLIST.index("var1")]) == 0:
lastfile = False
# Merging streams wrt time with no interpolation:
#ace_data = merge_ACE(ace_stream1, ace_stream2, merge_variables)
ace_data = mergeStreams(ace_stream1, ace_stream2, keys=merge_variables)
print (ace_data.length()[0])
if lastfile:
append == True
for key in merge_variables+['var1','var2','var3']:
keyind = KEYLIST.index(key)
if len(ace_last.ndarray[keyind]) == 0:
if debug:
print (keyind, len(ace_last.ndarray[keyind]))
print("Error in data - not appending.")
append == False
if append:
ace_data = appendStreams([ace_last, ace_data])
print (ace_data.length())
if not debug:
if newday == True:
print ("Created new {} file. Writing last data to yesterday.".format(ace_P))
ace_data.write(os.path.join(localpath,'collected'),filenamebegins="ace_%s_" % ace_P,
format_type='PYCDF', skipcompression=skipcompression)
else:
ace_data = ace_data.trim(starttime=today+"T00:00:00")
ace_data.write(os.path.join(localpath,'collected'),filenamebegins="ace_%s_" % ace_P,
format_type='PYCDF', skipcompression=skipcompression) # XXX
return True
