def connectclient(broker='localhost',
port=1883,
timeout=60,
credentials='',
user='',
password='',
qos=0):
"""
connectclient method
used to connect to a specific client as defined by the input variables
eventually add multiple client -> {"clients":[{"broker":"192.168.178.42","port":"1883"}]} # json type
import json
altbro = json.loads(altbrocker)
"""
clientid = "{}".format(broker)
client = mqtt.Client(clientid, False)
# Authentication part
if not credentials in ['', '-']:
# use user and pwd from credential data if not yet set
if user in ['', None, 'None', '-']:
user = mpcred.lc(credentials, 'user')
if password in ['', '-']:
password = mpcred.lc(credentials, 'passwd')
if not user in ['', None, 'None', '-']:
#client.tls_set(tlspath) # check http://www.steves-internet-guide.com/mosquitto-tls/
client.username_pw_set(
user, password=password
) # defined on broker by mosquitto_passwd -c passwordfile user
client.on_connect = on_connect
# on message needs: stationid, destination, location
client.on_message = on_message
client.connect(broker, port, timeout)
return client
def __init__(self, client, sensordict, confdict):
self.client = client #self.wsMcuFactory = wsMcuFactory
self.sensordict = sensordict
self.confdict = confdict
self.count = 0 ## counter for sending header information
self.sensor = sensordict.get('sensorid')
self.hostname = socket.gethostname()
self.printable = set(string.printable)
self.datalst = []
self.datacnt = 0
self.metacnt = 10
self.sensorlist = []
self.revision = self.sensordict.get('revision','')
try:
self.requestrate = int(self.sensordict.get('rate','-'))
except:
self.requestrate = 30
self.deltathreshold = confdict.get('timedelta')
# debug mode
debugtest = confdict.get('debug')
self.debug = False
if debugtest == 'True':
log.msg(' DEBUG - {}: Debug mode activated.'.format(self.sensordict.get('protocol')))
self.debug = True # prints many test messages
else:
log.msg(' -> Debug mode = {}'.format(debugtest))
# QOS
self.qos=int(confdict.get('mqttqos',0))
if not self.qos in [0,1,2]:
self.qos = 0
log.msg(" -> setting QOS:", self.qos)
# Database specific
self.db = self.sensor
# get existing sensors for the relevant board
log.msg(" -> IMPORTANT: MySQL assumes that database credentials ")
log.msg(" are saved locally using magpy.opt.cred with the same name as database")
try:
self.db = mdb.mysql.connect(host=mpcred.lc(self.sensor,'host'),user=mpcred.lc(self.sensor,'user'),passwd=mpcred.lc(self.sensor,'passwd'),db=self.sensor)
self.connectionMade(self.sensor)
except:
self.connectionLost(self.sensor,"Database could not be connected - check existance/credentials")
return
sensorlist = self.GetDBSensorList(self.db, searchsql='')
self.sensor = ''
existinglist = acs.GetSensors(confdict.get('sensorsconf'),identifier='$')
# if there is a sensor in existinglist which is not an active sensor, then drop it
for sensdict in existinglist:
if sensdict.get('sensorid','') in sensorlist:
self.sensorlist.append(sensdict)
self.lastt = [None]*len(self.sensorlist)
def ConnectDB(creddb):
print(" Accessing data bank ...")
try:
db = mysql.connect(host=mpcred.lc(creddb, 'host'),
user=mpcred.lc(creddb, 'user'),
passwd=mpcred.lc(creddb, 'passwd'),
db=mpcred.lc(creddb, 'db'))
print(" ... success")
return (db)
except:
print(" ... failure - check your credentials")
sys.exit()
def ConnectDB(dbcred):
# Connect to test database
# ------------------------------------------------------------
dbname = mpcred.lc(dbcred,'db')
dbhost = mpcred.lc(dbcred,'host')
dbpasswd = mpcred.lc(dbcred,'passwd')
dbuser = mpcred.lc(dbcred,'user')
print (dbname,dbhost,dbuser)
try:
print("Connecting to DATABASE...")
db = mpdb.mysql.connect(host="localhost",user=dbuser,passwd=dbpasswd,db=dbname)
print("... success")
except:
print("... failed")
db = None
return db
def connectDB(cred, exitonfailure=True, report=True):
if report:
print(" Accessing data bank... ")
try:
db = mysql.connect(host=mpcred.lc(cred, 'host'),
user=mpcred.lc(cred, 'user'),
passwd=mpcred.lc(cred, 'passwd'),
db=mpcred.lc(cred, 'db'))
if report:
print(" -> success. Connected to {}".format(mpcred.lc(
cred, 'db')))
except:
if report:
print(" -> failure - check your credentials / databank")
if exitonfailure:
sys.exit()
return db
def CheckConfiguration(config={}, debug=False):
"""
DESCRIPTION
configuration data will be checked
"""
user = ''
password = ''
address = ''
destination = ''
source = ''
port = 21
success = True
if debug:
print(" Checking configuration data")
if config.get('rawpath') == '' and creddb == '':
print(
'Specify either a shortcut to the credential information of the database or a local path:'
)
print('-- check collectfile.py -h for more options and requirements')
success = False
#sys.exit()
if config.get('rawpath') == '':
destination = tempfile.gettempdir()
else:
if not os.path.isdir(config.get('rawpath')):
print("Destination directory {} not existing. Creating it".format(
config.get('rawpath')))
os.makedirs(config.get('rawpath'))
destination = config.get('rawpath')
config['destination'] = destination
credtransfer = config.get('sourcecredentials')
if not credtransfer == '':
if debug:
print(" - checking credentials for remote access")
user = mpcred.lc(credtransfer, 'user')
password = mpcred.lc(credtransfer, 'passwd')
address = mpcred.lc(credtransfer, 'address')
try:
port = int(mpcred.lc(credtransfer, 'port'))
except:
port = 21
if debug:
print(" -> done")
config['rmuser'] = user
config['rmpassword'] = password
config['rmaddress'] = address
config['rmport'] = port
source = ''
protocol = config.get('protocol')
if not protocol in ['', 'ftp', 'FTP']:
source += protocol + "://"
if not user == '' and not password == '':
source += user + ":" + password + "@"
if not address == '':
source += address
remotepath = config.get('sourcedatapath')
if not remotepath == '':
source += remotepath
config['source'] = source
if not protocol in ['', 'scp', 'ftp', 'SCP', 'FTP', 'html', 'rsync']:
print('Specify a valid protocol:')
print('-- check collectfile.py -h for more options and requirements')
success = False
#sys.exit()
walk = config.get('walksubdirs')
if debug:
print(" Walk through subdirs: {}".format(walk))
if GetBool(walk):
if not protocol in ['', 'scp', 'rsync']:
print(
' -> Walk mode only works for local directories and scp access.'
)
print(' -> Switching walk mode off.')
config['walksubdirs'] = False
creddb = config.get('dbcredentials')
if not creddb == '':
print(" Accessing data bank ...")
# required for either writeing to DB or getting meta in case of writing archive
try:
db = mysql.connect(host=mpcred.lc(creddb, 'host'),
user=mpcred.lc(creddb, 'user'),
passwd=mpcred.lc(creddb, 'passwd'),
db=mpcred.lc(creddb, 'db'))
print(" -> success")
except:
print(" -> failure - check your credentials")
db = None
success = False
#sys.exit()
config['db'] = db
# loaded all credential (if started from root rootpermissions are relquired for that)
# now switch user for scp
# TODO check whether this is working in a function
if config.get('defaultuser'):
try:
uid = pwd.getpwnam(config.get('defaultuser'))[2]
os.setuid(uid)
except:
print(" User {} not existing - moving on".format(
config.get('defaultuser')))
dateformat = config.get('dateformat')
filename = config.get('filenamestructure')
if dateformat == "" and filename == "":
print(
' Specify either a fileformat: -f myformat.dat or a dateformat -d "%Y",ctime !'
)
print(
' -- check collectfile.py -h for more options and requirements')
success = False
#sys.exit()
if not dateformat in ['', 'ctime', 'mtime']:
current = datetime.utcnow()
try:
newdate = datetime.strftime(current, dateformat)
except:
print(' Specify a vaild datetime dateformat like "%Y-%m-%d"')
print(
' -- check collectfile.py -h for more options and requirements'
)
success = False
#sys.exit()
if "%s" in filename and dateformat in ['', 'ctime', 'mtime']:
print(
' Specify a datetime dateformat for given placeholder in fileformat!'
)
print(
' -- check collectfile.py -h for more options and requirements')
success = False
#sys.exit()
elif not "%s" in filename and "*" in filename and not dateformat in [
'ctime', 'mtime'
]:
print(
' Specify either ctime or mtime for dateformat to be used with your give fileformat!'
)
print(
' -- check collectfile.py -h for more options and requirements')
success = False
#sys.exit()
elif not "%s" in filename and not "*" in filename and not dateformat in [
""
]:
print(' Give dateformat will be ignored!')
print(
' -- check collectfile.py -h for more options and requirements')
print(' -- continuing ...')
if debug:
print(" => Configuration checked - success")
return config, success
name = "{}-DataProducts-magnetism".format(sn)
try:
from magpy.opt.analysismonitor import *
analysisdict = Analysismonitor(
logfile='/home/cobs/ANALYSIS/Logs/AnalysisMonitor_cobs.log')
analysisdict = analysisdict.load()
except:
print("Analysis monitor failed")
pass
# ################################################
# Database connection
# ################################################
dbpasswd = mpcred.lc('cobsdb', 'passwd')
try:
# Test MARCOS 1
print("Connecting to primary MARCOS...")
db = mysql.connect(host="138.22.188.195",
user="******",
passwd=dbpasswd,
db="cobsdb")
except:
print("... failed")
try:
# Test MARCOS 2
print("Connecting to secondary MARCOS...")
db = mysql.connect(host="138.22.188.191",
user="******",
passwd=dbpasswd,
import magpy.opt.cred as mpcred
ippath = '/home/cobs/MARTAS/Logs/ip.txt'
last = open(ippath, 'r')
lastip = last.read()
last.close()
#sendcmd = 'ifconfig > ' + ippath
sendcmd = 'wget -qO- http://ipecho.net/plain > ' + ippath
subprocess.call(sendcmd, shell=True)
cred = 'zamg'
sendpath = '/data/station1'
address = mpcred.lc(cred, 'address')
user = mpcred.lc(cred, 'user')
passwd = mpcred.lc(cred, 'passwd')
port = 21
pathtolog = '/tmp/magpytransfer.log'
current = open(ippath, 'r')
currentip = current.read()
current.close()
if currentip != lastip:
print "IP has changed. Sending..."
ftpdatatransfer(localfile=ippath,
ftppath=sendpath,
myproxy=address,
port=port,
def main(argv):
##
## Run like: python acquisition.py -m '/home/cobs/MARTAS/defaults.conf'
global now
global hostname
global msgcount
global SUPPORTED_PROTOCOLS
passive_count = 0
active_count = 0
martasfile = 'martas.cfg'
cred = ''
creduser = ''
credhost = ''
pwd = 'None'
## Get eventually provided options
## ----------------------------
usagestring = 'acquisition.py -m <martas> -c <credentials> -P <password>'
try:
opts, args = getopt.getopt(argv, "hm:c:P:U", [
"martas=",
"credentials=",
"password="******"debug=",
])
except getopt.GetoptError:
print('Check your options:')
print(usagestring)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('------------------------------------------------------')
print('Usage:')
print(usagestring)
print('------------------------------------------------------')
print('Options:')
print('-h help')
print(
'-m path to martas configuration')
print(
'-c credentials, if authentication is used'
)
print(
'-P alternatively provide password'
)
print('------------------------------------------------------')
print('Examples:')
print('1. Basic (using defauilt martas.cfg')
print(' python acquisition.py')
print('2. Using other configuration')
print(' python acquisition.py -m "/home/myuser/mymartas.cfg"')
sys.exit()
elif opt in ("-m", "--martas"):
martasfile = arg
elif opt in ("-c", "--credentials"):
cred = arg
elif opt in ("-P", "--password"):
pwd = arg
## Load defaults dict
## ----------------------------
conf = acs.GetConf(martasfile)
# Add a ceck routine here whether conf information was obtained
broker = conf.get('broker')
mqttport = int(conf.get('mqttport'))
mqttdelay = int(conf.get('mqttdelay'))
## Get Sensor data
## ----------------------------
sensorlist = acs.GetSensors(conf.get('sensorsconf'))
## Check for credentials
## ----------------------------
if not cred == '':
try:
print("Accessing credential information for {}".format(cred))
credpath = conf.get('credentialpath', None)
credhost = mpcred.lc(cred, 'address', path=credpath)
creduser = mpcred.lc(cred, 'user', path=credpath)
pwd = mpcred.lc(cred, 'passwd', path=credpath)
except:
print("error when accessing credentials")
pass
## create MQTT client
## ----------------------------
client = mqtt.Client(clean_session=True)
user = conf.get('mqttuser', '')
if not user in ['', '-', None, 'None']:
# Should have two possibilities:
# 1. check whether credentials are provided
if not cred == '':
if not creduser == user:
print(
'User names provided in credentials ({}) and martas.cfg ({}) differ. Please check!'
.format(creduser, user))
pwd = 'None'
if pwd == 'None':
# 2. request pwd input
print('MQTT Authentication required for User {}:'.format(user))
import getpass
pwd = getpass.getpass()
client.username_pw_set(username=user, password=pwd)
## Start Twisted logging system
## ----------------------------
if conf.get('logging').strip() == 'sys.stdout':
log.startLogging(sys.stdout)
else:
try:
print(" -- Logging to {}".format(conf.get('logging')))
log.startLogging(open(conf.get('logging'), 'a'))
log.msg("----------------")
log.msg(" -> Logging to {}".format(conf.get('logging')))
except:
log.startLogging(sys.stdout)
print("Could not open {}. Switching log to stdout.".format(
conf['logging']))
log.msg("----------------")
log.msg("Starting MARTAS acquisition version {}".format(__version__))
log.msg("----------------")
## connect to MQTT client
## ----------------------------
client.on_connect = onConnect
try:
client.connect(broker, mqttport, mqttdelay)
client.loop_start()
except:
log.msg(
"Critical error - no network connection available during startup or mosquitto server not running - check whether data is recorded"
)
establishedconnections = {}
## Connect to serial port (sensor dependency) -> returns publish
# Start subprocesses for each publishing protocol
for sensor in sensorlist:
log.msg("----------------")
log.msg("Sensor and Mode:", sensor.get('sensorid'), sensor.get('mode'))
log.msg("----------------")
init = sensor.get('init')
if not init in ['', 'None', None, 0, '-']:
log.msg(" - Initialization using {}".format(init))
initdir = conf.get('initdir')
initapp = os.path.join(initdir, init)
# Check if provided initscript is existing
import subprocess
try:
log.msg(" - running initialization {}".format(initapp))
initcall = "{} {}".format(sys.executable, initapp)
#log.msg(subprocess.check_output(['/bin/sh',initapp]))
log.msg(subprocess.check_output(initcall))
except subprocess.CalledProcessError as e:
log.msg(
" - init command '{}' returned with error (code {}): {}".
format(e.cmd, e.returncode, e.output))
except:
pass
if sensor.get('mode') in ['p', 'passive', 'Passive', 'P']:
try:
connected = PassiveThread(conf, sensor, client,
establishedconnections)
log.msg(
" - PassiveThread initiated for {}. Ready to receive data ..."
.format(sensor.get('sensorid')))
establishedconnections.update(connected)
passive_count += 1
except:
log.msg(" - !!! PassiveThread failed for {} !!!".format(
sensor.get('sensorid')))
pass
elif sensor.get('mode') in ['a', 'active', 'Active', 'A']:
try:
log.msg(
" - ActiveThread initiated for {}. Periodically requesting data ..."
.format(sensor.get('sensorid')))
connected_act = ActiveThread(conf, sensor, client,
establishedconnections)
except:
log.msg(" - !!! ActiveThread failed for {} !!!".format(
sensor.get('sensorid')))
pass
elif sensor.get('mode') in ['autonomous']:
try:
log.msg(
" - AutoThread initiated for {}. Ready to receive data ..."
.format(sensor.get('sensorid')))
connected_act = AutoThread(conf, sensor, client,
establishedconnections)
except Exception as e:
log.msg(" - !!! AutoThread failed for {} !!!".format(
sensor.get('sensorid')))
log.msg(e)
pass
else:
log.msg("acquisition: Mode not recognized")
sensorid = sensor.get('sensorid')
# Start all passive clients
if passive_count > 0:
log.msg(
"acquisition: Starting reactor for passive sensors. Sending data now ..."
)
reactor.run()
# TODO other solution - when the main thread stops, the deamons are killed!
got_here = True
print("this is the end of the main thread...")
if got_here:
import time
while True:
time.sleep(100)
def main(argv):
cred = ''
path = ''
remotepath = ''
protocol = ''
depth = 2
increment = False
extension = 'bin'
dateformat = '%Y-%m-%d'
compress = False
try:
opts, args = getopt.getopt(argv, "hc:l:r:s:d:i:e:f:z", [
"cred=",
"localpath=",
"remotepath=",
"protocol=",
"depth=",
"increment=",
"extension=",
"dateformat=",
])
except getopt.GetoptError:
print 'senddata.py -c <credentialshortcut> -s <protocol> -l <localpath> -r <remotepath> -d <depth> -i <increment> -e <extension> -f <dateformat> -z'
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print '-------------------------------------'
print 'Description:'
print 'Sending data to a remote host by scp or ftp.'
print 'Requires existing credential information (see cred.py).'
print '-------------------------------------'
print 'Usage:'
print 'senddata.py -c <credentialshortcut> -l <localpath> -r <remotepath> '
print ' -s <protocol> -d <depth> -i <increment> -e <extension> -f <dateformat>'
print '-------------------------------------'
print 'Options:'
print '-c (required): the credentials for the transfer protocol.'
print ' Create using the addcred.py method. Use'
print ' python addred.py -h for help on that.'
print '-l (required): provide a local root path, all directories below will'
print ' be scanned for *date.bin files'
print '-r : eventually provide a remote path like "/data"'
print '-s (required): provide the protocol to be used (ftp or scp)'
print '-d : defines the amount of days to be send: 1 for today only,'
print ' 2 for the last two days and so on. d needs to be a integer'
print ' with d > 1.'
print '-i (experimental): defines incremental uploads. Loaded data is extracted by Magpy'
print ' and i minutes are uploaded and appended to an existing file.'
print ' This file needs to be unified later on.'
print '-e : provide a user defined extension, default is "bin"'
print '-f : provide a date format, default is "%Y-%m-%d"'
print '-z : compress file before sending'
print '-------------------------------------'
print 'Examples:'
print 'python senddata.py -c zamg -s ftp -l /srv/ws/ -r /data'
sys.exit()
elif opt in ("-c", "--cred"):
cred = arg
elif opt in ("-l", "--localpath"):
path = arg
elif opt in ("-r", "--remotepath"):
remotepath = arg
elif opt in ("-s", "--protocol"):
protocol = arg
elif opt in ("-d", "--depth"):
try:
depth = int(arg)
if not depth >= 1:
print "depth needs to be positve"
sys.exit()
except:
print "depth needs to be an integer"
sys.exit()
elif opt in ("-i", "--increment"):
increment = arg
elif opt in ("-e", "--extension"):
extension = arg.strip('.')
elif opt in ("-f", "--dateformat"):
dateformat = arg
elif opt in ("-z", "--compress"):
compress = True
if cred == '':
print 'Specify a shortcut to credentials. '
print '-- use addcred.py for this purpose.'
print '-- check senddata.py -h for more options and requirements'
sys.exit()
if path == '':
print 'Specify a base path. '
print '-- check senddata.py -h for more options and requirements'
sys.exit()
if protocol == '':
print 'Specify a protocol (scp, ftp). '
print '-- check senddata.py -h for more options and requirements'
sys.exit()
# Test with missing information
address = mpcred.lc(cred, 'address')
user = mpcred.lc(cred, 'user')
passwd = mpcred.lc(cred, 'passwd')
port = mpcred.lc(cred, 'port')
pathtolog = '/tmp/magpytransfer.log'
if increment:
depth = 1
datelist = []
current = datetime.utcnow()
newcurrent = current
for elem in range(depth):
datelist.append(datetime.strftime(newcurrent, dateformat))
newcurrent = current - timedelta(days=elem + 1)
print datelist
for date in datelist:
for dirpath, dirnames, filenames in os.walk(path):
for filename in [
f for f in filenames if f.endswith(date + "." + extension)
]:
localfile = os.path.join(dirpath, filename)
print "Sending ", localfile
if compress:
print 'creating compressed archive'
zfile = os.path.join(dirpath,
filename.strip(extension) + 'zip')
zf = zipfile.ZipFile(zfile, mode='w')
try:
zf.write(localfile,
os.path.basename(localfile),
compress_type=compression)
finally:
print 'closing'
zf.close()
localfile = zfile
print "Sending ", localfile
if protocol == 'ftp':
# Tested - working flawless (take care with address - should not contain ftp://)
ftpdatatransfer(localfile=localfile,
ftppath=remotepath,
myproxy=address,
port=port,
login=user,
passwd=passwd,
logfile=pathtolog,
raiseerror=True)
pass
elif protocol == 'scp':
# Tested - working flawless
scptransfer(
localfile, user + '@' + address + ':' + remotepath +
'/' + filename, passwd)
pass
elif protocol == 'gin':
print "GIN not supported yet"
# Coming soon
sys.exit()
else:
print "Unsupported protocol selected:"
print '-- Specify one among (scp, ftp). '
print '-- check senddata.py -h for more options and requirements'
sys.exit()
print "... success"
def sendmail(dic):
"""
Function for sending mails with attachments
"""
#if not smtpserver:
# smtpserver = 'smtp.web.de'
if 'Attach' in dic:
files = map(lambda s: s.strip(), dic['Attach'].split(','))
else:
files = []
if not dic['Text']:
text = 'Cheers, Your Analysis-Robot'
if not 'Subject' in dic:
dic['Subject'] = 'Automatic message'
if 'mailcred' in dic:
## import credential routine
import magpy.opt.cred as cred
#read credentials
print("Reading credentials")
dic['smtpserver'] = cred.lc(dic.get('mailcred'), 'smtp')
dic['user'] = cred.lc(dic.get('mailcred'), 'user')
dic['pwd'] = cred.lc(dic.get('mailcred'), 'passwd')
#dic['port'] = cred.lc(dic.get('mailcred'),'port') ## port is currently not stored by addcred
if 'port' in dic:
port = int(dic['port'])
else:
port = None
if 'user' in dic:
user = dic['user']
else:
user = ''
msg = MIMEMultipart()
msg['From'] = dic['From']
send_from = dic['From']
#msg['To'] = COMMASPACE.join(send_to)
msg['To'] = dic['To']
if len(dic['To'].split(',')) > 1:
send_to = list(map(lambda s: s.strip(), dic['To'].split(',')))
else:
send_to = dic.get('To').strip()
msg['Date'] = formatdate(localtime=True)
msg['Subject'] = dic['Subject']
msg.attach(MIMEText(dic['Text']))
# TODO log if file does not exist
for f in files:
part = MIMEBase('application', "octet-stream")
part.set_payload(open(f, "rb").read())
encoders.encode_base64(part)
part.add_header('Content-Disposition',
'attachment; filename="%s"' % os.path.basename(f))
msg.attach(part)
# seems as if server name needs to be specified in py3.7 and 3.8, should work in older versions as well
smtp = SMTP(dic.get('smtpserver'))
smtp.set_debuglevel(False)
if port:
smtp.connect(dic.get('smtpserver'), port)
else:
smtp.connect(dic.get('smtpserver'))
smtp.ehlo()
if port == 587:
smtp.starttls()
smtp.ehlo()
if user:
smtp.login(user, dic.get('pwd'))
smtp.sendmail(send_from, send_to, msg.as_string())
smtp.close()
#!/usr/bin/env python
"""
Skeleton for graphs
"""
from magpy.stream import *
from magpy.database import *
from magpy.transfer import *
import magpy.mpplot as mp
import magpy.opt.emd as emd
import magpy.opt.cred as mpcred
dbpasswd = mpcred.lc('cobsdb','passwd')
try:
# Test MARCOS 1
print "Connecting to primary MARCOS..."
db = mysql.connect(host="localhost",user="******",passwd=dbpasswd,db="cobsdb")
print db
except:
print "... failed"
try:
# Test MARCOS 2
print "Connecting to secondary MARCOS..."
db = mysql.connect(host="138.22.188.191",user="******",passwd=dbpasswd,db="cobsdb")
print db
except:
print "... failed -- aborting"
sys.exit()
from magpy.database import *
from magpy.transfer import *
import magpy.mpplot as mp
import magpy.opt.emd as emd
import magpy.opt.cred as mpcred
import itertools
from threading import Thread
import json
from martas import martaslog as ml
logpath = '/var/log/magpy/mm-dp-autodif.log'
sn = 'SAGITTARIUS'
statusmsg = {}
dbpasswd = mpcred.lc('cobsdb', 'passwd')
try:
# Test MARCOS 1
print("Connecting to primary MARCOS...")
db = mysql.connect(host="138.22.188.195",
user="******",
passwd=dbpasswd,
db="cobsdb")
except:
print "... failed"
try:
# Test MARCOS 2
print("Connecting to secondary MARCOS...")
db = mysql.connect(host="138.22.188.191",
user="******",
passwd=dbpasswd,
def main(argv):
creddb = ''
credtransfer = ''
user=''
password= ''
address = ''
port = 21
protocol = ''
localpath = ''
remotepath = ''
sensorid = ''
stationid = ''
startdate = ''
dateformat = ''
depth = 1
filename = ''
db = False
disableproxy=False
zipping = False
walk=False
defaultuser = ''
uppercase=False
force = ''
forcelist = ['0001','0002','0003','0004','0005','0006','0007','0008']
debug = False
try:
opts, args = getopt.getopt(argv,"hc:e:l:r:p:s:t:b:d:a:f:Uowu:xm:z",["creddb=","credtransfer=","localpath=","remotepath=","protocol=","sensorid=","stationid=","startdate=","depth=","dateformat=", "filefomat=","debug=","disableproxy=","walk=","user="******"uppercase=","insert-table=","zip="])
except getopt.GetoptError:
print('collectfile.py -c <creddb> -e <credtransfer> -l <localpath> -r <remotepath> -p <protocol> -s <sensorid> -t <stationid> -b <startdate> -d <depth> -a <dateformat> -f <filefomat> -U <debug> -o <disableproxy=True> -w <walk=True> -u <user> -x <uppercase> -m <insert-table> -z <zip>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('-------------------------------------')
print('Description:')
print('collectfile.py reads data from various sources ')
print('and uploads data to a data bank.')
print('Filtering and archiving is done using "cleanup".')
print('-------------------------------------')
print('Usage:')
print('collectfile.py -c <creddb> -e <credtransfer> -l <localpath> -r <remotepath> -p <protocol> -s <sensorid> -t <stationid> -b <startdate> -d <depth> -a <dateformat> -f <filefomat> -U <debug> -o <disableproxy=True> -w <walk=True> -u <user> -x <uppercase> -m <insert-table> -z <zip>')
print('-------------------------------------')
print('Options:')
print('-c : provide the shortcut to the data bank credentials')
print('-e : credentials for transfer protocol')
print('-l : localpath - if provided, raw data will be stored there.')
print(' : two subdirectories will be created - stationID and sensorID')
print('-r (required) : remotepath - path to the data to be collected')
print('-p : protocol of data access - required for ftp and scp')
print('-s : ID of the sensor (required if not contained in the data')
print(' meta information)')
print('-t : ID of the station i.e. the Observatory code (required if')
print(' not in meta data)')
print('-b : date to start with, like 2014-11-22, default is current day')
print('-d : depth: 1 means today, 2 today and yesterday, 3 last three days, etc')
print('-a : dateformat in files to be read')
print(' like "%Y-%m-%d" for 2014-02-01')
print(' "%Y%m%d" for 20140201')
print(' "ctime" or "mtime" for using timestamp of file')
print(' Check out pythons datetime function for more info')
print('-f : filename of data file to be read.')
print(' Add %s as placeholder for date')
print(' examples: "WIC_%s.bin"')
print(' "*%s*"')
print(' "WIC_%s.*"')
print(' "WIC_2013.all" - no dateformat -> single file will be read')
print('-o (no input) : if selected any systems proxy settings are disabled')
print('-w (no input) : if selected all subdirectories below remote path will be searched for')
print(' filename pattern. Only works for local directories and scp.')
print('-u : perform upload as this user - necessary for cron and other root jobs')
print(' as root cannot use scp transfer.')
print('-x : use uppercase for dateformat (e.g. NOV2014 instead of Nov2014)')
print('-m : force data to the given revision number.')
print('-z : if option l is selected raw data will be zipped within localpath.')
print('-------------------------------------')
print('Examples:')
print('---------')
print('1. get data from ftp server and add to database')
print(' python collectfile.py -r "/data/magnetism/gam" -c cobsdb -e zamg -p ftp -t GAM ')
print(' -d 2 -f *%s.zip -a "%Y-%m-%d"')
print('---------')
print('2. get data from local directory and add to database')
print('python collectfile.py -c cobsdb -r "/srv/data/"')
print(' -s LEMI036_1_0001 -t WIC -a "%Y-%m-%d" -f "LEMI036_1_0001_%s.bin" ')
print('---------')
print('3. get data from local directory and add to database, add raw data to archive')
print('python collectfile.py -c cobsdb -r "/Observatory/archive/WIK/DIDD_3121331_0002/DIDD_3121331_0002_0001/" -s DIDD_3121331_0002 -t WIK -b "2012-06-01" -d 100 -a "%Y-%m-%d" -f "DIDD_3121331_0002_0001_%s.cdf" -l "/srv/archive"')
print('---------')
print('4. get data from remote by ssh and store in local archive')
print('python collectfile.py -e phobostilt -r "/srv/gwr/" -p scp -s GWRSG_12345_0002 -t SGO -b "2012-06-01" -d 30 -a "%y%m%d" -f "G1%s.025" -l "/srv/archive"')
print('---------')
print('5. get recently created files from remote by ssh and store in local archive')
print('python collectfile.py -e themisto -r "/srv/" -p scp -t SGO -d 2 -a ctime -l "/srv/archive"')
print('---------')
sys.exit()
elif opt in ("-c", "--creddb"):
creddb = arg
elif opt in ("-e", "--credtransfer"):
credtransfer = arg
elif opt in ("-l", "--localpath"):
localpath = arg
elif opt in ("-p", "--protocol"):
protocol = arg
elif opt in ("-r", "--remotepath"):
remotepath = arg
elif opt in ("-s", "--sensorid"):
sensorid = arg
elif opt in ("-t", "--stationid"):
stationid = arg
elif opt in ("-d", "--depth"):
try:
depth = int(arg)
if not depth >= 1:
print("depth needs to be positve")
sys.exit()
except:
print("depth needs to be an integer")
sys.exit()
elif opt in ("-a", "--dateformat"):
dateformat = arg
elif opt in ("-b", "--begin"):
startdate = arg
elif opt in ("-f", "--filename"):
filename = arg
elif opt in ("-o", "--option"):
disableproxy=True
elif opt in ("-w", "--walk"):
walk=True
elif opt in ("-u", "--user"):
defaultuser = arg
elif opt in ("-x", "--uppercase"):
uppercase=True
elif opt in ("-z", "--zip"):
zipping=True
elif opt in ("-m", "--insert-table"):
force=arg
if not force in forcelist:
print ("-m: provided data revision number is not valid: should be 0001 or similar")
force = ''
elif opt in ("-U", "--debug"):
debug = True
### ###############################################################################
### 1. Check input variables
### ###############################################################################
if localpath == '' and creddb == '':
print('Specify either a shortcut to the credential information of the database or a local path:')
print('-- check collectfile.py -h for more options and requirements')
sys.exit()
if localpath == '':
destination = tempfile.gettempdir()
else:
if not os.path.isdir(localpath):
print ("Destination directory {} not existing. Creating it".format(localpath))
os.makedirs(localpath)
destination = localpath
if not credtransfer == '':
user=mpcred.lc(credtransfer,'user')
password=mpcred.lc(credtransfer,'passwd')
address = mpcred.lc(credtransfer,'address')
try:
port = int(mpcred.lc(credtransfer,'port'))
except:
port = 21
source = ''
if not protocol in ['','ftp','FTP']:
source += protocol + "://"
if not user == '' and not password=='':
source += user + ":" + password + "@"
if not address == '':
source += address
if not remotepath == '':
source += remotepath
if not protocol in ['','scp','ftp','SCP','FTP','html','rsync']:
print('Specify a valid protocol:')
print('-- check collectfile.py -h for more options and requirements')
sys.exit()
if walk:
if not protocol in ['','scp','rsync']:
print(' Walk mode only works for local directories and scp access.')
print(' Switching walk mode off.')
walk = False
if not creddb == '':
print("Accessing data bank ...")
try:
db = mysql.connect (host=mpcred.lc(creddb,'host'),user=mpcred.lc(creddb,'user'),passwd=mpcred.lc(creddb,'passwd'),db =mpcred.lc(creddb,'db'))
print("success")
except:
print("failure - check your credentials")
sys.exit()
# loaded all credential (if started from root rootpermissions are relquired for that)
# now switch user for scp
if not defaultuser == '':
uid=pwd.getpwnam(defaultuser)[2]
os.setuid(uid)
if startdate == '':
current = datetime.utcnow() # make that a variable
else:
current = DataStream()._testtime(startdate)
if dateformat == "" and filename == "":
print('Specify either a fileformat: -f myformat.dat or a dateformat -d "%Y",ctime !')
print('-- check collectfile.py -h for more options and requirements')
sys.exit()
if not dateformat in ['','ctime','mtime']:
try:
newdate = datetime.strftime(current,dateformat)
except:
print('Specify a vaild datetime dateformat like "%Y-%m-%d"')
print('-- check collectfile.py -h for more options and requirements')
sys.exit()
if "%s" in filename and dateformat in ['','ctime','mtime']:
print('Specify a datetime dateformat for given placeholder in fileformat!')
print('-- check collectfile.py -h for more options and requirements')
sys.exit()
elif not "%s" in filename and "*" in filename and not dateformat in ['ctime','mtime']:
print('Specify either ctime or mtime for dateformat to be used with your give fileformat!')
print('-- check collectfile.py -h for more options and requirements')
sys.exit()
elif not "%s" in filename and not "*" in filename and not dateformat in [""]:
print('Give dateformat will be ignored!')
print('-- check collectfile.py -h for more options and requirements')
print('-- continuing ...')
if debug:
print("1.0 finished - Parameters OK")
### ###############################################################################
### 2. Create download/copy link
### ###############################################################################
### for all files conform with eventually provided datelist
### 2.1 Check dates
### -------------------------------------
### the following parameters are used here:
### dateformat, filename
### dateformat: string like ("%Y-%m-%d"), ("ctime", "mtime") or ""
### filename: string like "*%s.bin" (-> requires dateformat[0] )
### string like "*.bin" (-> requires dateformat[1] )
### string like "myfile.bin" (-> requires dateformat[2] )
### empty "" (-> searches either for dateformat[0] or takes all fitting with dateformat[2] )
### -> see 1. check parameter
### make use of depth and begin to define timerange
datelist = []
newcurrent = current
if not dateformat in ['','ctime','mtime']:
for elem in range(depth):
if dateformat == '%b%d%y': #exception for MAGREC
newdate = datetime.strftime(newcurrent,dateformat)
datelist.append(newdate.upper())
else:
datelist.append(datetime.strftime(newcurrent,dateformat))
newcurrent = current-timedelta(days=elem+1)
elif dateformat in ['ctime','mtime']:
for elem in range(depth):
datelist.append(newcurrent)
newcurrent = current-timedelta(days=elem+1)
else:
datelist = ['dummy']
#if debug:
print(" - Dealing with time range:\n {}".format(datelist))
### 2.2 Select files from source meeting critera
### -------------------------------------
### Define source based on 'protocol', 'remotepath', 'walk', 'option' and optionally 'sensorid'
### protocols: ''(local disk), 'scp', 'ftp', 'html'
#filelist = getfilelist(protocol, source, sensorid, filename, datelist, walk=True, option=None)
if debug:
print("2.2 Starting - Getting filelists")
filelist = []
if protocol in ['ftp','FTP']:
if debug:
print (" - Getting filelist - by ftp ")
import ftplib
if debug:
print (" - connecting to {} on port {}".format(address,port))
if not port == 21:
ftp = ftplib.FTP()
ftp.connect(address,port)
else:
ftp = ftplib.FTP(address)
if debug:
print (" - user: {} ".format(user))
ftp.login(user,password)
ftp.cwd(source)
lines = []
ftp.dir("", lines.append)
ftp.close()
for date in datelist:
path = dir_extract(lines, filename, date, dateformat)
if len(path) > 0:
filelist.extend(path)
elif protocol in ['scp','SCP','rsync']:
if debug:
print (" - Getting filelist - by ssh ")
pwd_required=True
if protocol == 'rsync':
pwd_required=False
print ("Rsync requires passwordless ssh connection to remote system")
import pexpect
if not dateformat in ['','ctime','mtime']:
for date in datelist:
path = ssh_getlist(remotepath, filename, date, dateformat, datetime.utcnow(), cred=[user,password,address],pwd_required=pwd_required)
if len(path) > 0:
filelist.extend(path)
else:
filelist = ssh_getlist(remotepath, filename, min(datelist), dateformat, max(datelist), cred=[user,password,address],pwd_required=pwd_required)
elif protocol == '':
if debug:
print (" - Getting filelist - from local directory ")
### Search local directory - Working
for date in datelist:
path = walk_dir(source, filename, date, dateformat)
if len(path) > 0:
filelist.extend(path)
elif protocol == 'html':
print (filelist)
sys.exit()
if debug:
print ("Result")
print ("-----------------------------")
print (filelist)
### 2.3 Get selected files and copy them to destination
### -------------------------------------
###
### only if not protocol == '' and localpath
### update filelist with new filenamens on local harddisk
if debug:
print("2.3 Writing data to a local directory (or tmp)")
localpathlist = []
if not protocol == '' or (protocol == '' and not destination == tempfile.gettempdir()):
### Create a directory by getting sensorid names (from source directory)
def createdestinationpath(localpath,stationid,sensorid):
subdir = 'raw'
if not stationid and not sensorid:
destpath = os.path.join(localpath)
elif not stationid:
destpath = os.path.join(localpath,sensorid,'raw')
elif not sensorid:
destpath = os.path.join(localpath,stationid.upper())
else:
destpath = os.path.join(localpath,stationid.upper(),sensorid,'raw')
return destpath
# Open the specific channel
if protocol in ['ftp','FTP']:
if not port == 21:
ftp = ftplib.FTP()
ftp.connect(address,port)
else:
ftp = ftplib.FTP(address)
ftp.login(user,password)
ftp.cwd(source)
for f in filelist:
path = os.path.normpath(f)
li = path.split(os.sep)
if not sensorid and not protocol in ['ftp','FTP']:
if len(li) >= 2:
sensid = li[-2]
elif not sensorid and protocol in ['ftp','FTP']:
sensid = f.split('.')[0].rpartition('_')[0]
else:
sensid = sensorid
destpath = createdestinationpath(destination,stationid,sensid)
destname = os.path.join(destpath,li[-1])
if not os.path.isdir(destpath):
os.makedirs(destpath)
if debug:
print ("DESTINATION (for files):", destpath, li[-1])
if protocol in ['ftp','FTP']:
fhandle = open(destname, 'wb')
ftp.retrbinary('RETR ' + f, fhandle.write)
fhandle.close()
elif protocol in ['scp','SCP']:
scptransfer(user+'@'+address+':'+f,destpath,password,timeout=600)
elif protocol in ['rsync']:
# create a command line string with rsync ### please note,,, rsync requires password less comminuctaion
rsyncstring = "rsync -avz -e ssh {} {}".format(user+'@'+address+':'+f,destpath)
print ("Executing:", rsyncstring)
subprocess.call(rsyncstring.split())
elif protocol in ['html','HTML']:
pass
elif protocol in ['']:
copyfile(f, destname)
if zipping:
if debug:
print (" raw data wil be zipped")
dirname = os.path.dirname(destname)
oldname = os.path.basename(destname)
pname = os.path.splitext(oldname)
if not pname[1] in [".zip",".gz",".ZIP",".GZ"]:
zipname = pname[0]+'.zip'
with zipfile.ZipFile(os.path.join(dirname,zipname), 'w') as myzip:
myzip.write(destname,oldname, zipfile.ZIP_DEFLATED)
os.remove(destname)
destname = os.path.join(dirname,zipname)
else:
if debug:
print (" data is zipped already")
localpathlist.append(destname)
if protocol in ['ftp','FTP']:
ftp.close()
else:
localpathlist = [elem for elem in filelist]
### ###############################################################################
### 3. Read local data and write to database
### ###############################################################################
### for all files conform with eventually provided datelist
### 3.1 Read local data
### -------------------------------------
### flagging does not make sense
if db:
if debug:
print("3.1 Writing data to database")
for f in localpathlist:
data = read(f)
if debug:
print ("Dealing with {}. Length = {}".format(f,data.length()[0]))
print ("SensorID in file: {}".format(data.header.get('SensorID')))
statiddata = data.header.get('StationID','')
if not stationid == '':
if not statiddata == stationid and not statiddata == '':
print("StationID's from file and provided one (or dir) are different!")
print ("Using provided value")
data.header['StationID'] = stationid
else:
if data.header.get('StationID','') == '':
print("Could not find station ID in datafile")
print("Please provide by using -t stationid")
sys.exit()
if debug:
print("Using StationID", data.header.get('StationID'))
sensiddata = data.header.get('SensorID','')
if not sensorid == '':
if not sensiddata == sensorid and not sensiddata == '':
print("SensorID's from file and provided one (or dir) are different!")
print ("Using provided value")
data.header['SensorID'] = sensorid
else:
if data.header.get('SensorID','') == '':
print("Could not find sensor ID in datafile")
print("Please provide by using -s sensorid")
sys.exit()
if debug:
print("Using SensorID", data.header.get('SensorID'))
print("{}: Adding {} data points to DB now".format(data.header.get('SensorID'), data.length()[0]))
if not len(data.ndarray[0]) > 0:
data = data.linestruct2ndarray() # Dealing with very old formats
if len(data.ndarray[0]) > 0:
if not force == '':
tabname = data.header.get('SensorID')+'_'+force
print (" - Force option chosen: forcing data to table {}".format(tabname))
writeDB(db,data, tablename=tabname)
else:
writeDB(db,data)
pass
- magpy
- sudo apt install percona-toolkit
- main user (cobs) needs to be able to use sudo without passwd (add to /etc/sudoers)
"""
from __future__ import print_function
# Define packges to be used (local refers to test environment)
# ------------------------------------------------------------
from magpy.stream import *
from magpy.database import *
import magpy.opt.cred as mpcred
# Get password from cred
# ------------------------------------------------------------
dbpasswd = mpcred.lc('cobsdb', 'passwd')
dbuser = mpcred.lc('cobsdb', 'user')
sqluser = mpcred.lc('sql', 'user')
sqlpwd = mpcred.lc('sql', 'passwd')
# Use Telegram logging
# ------------------------------------------------------------
logpath = '/var/log/magpy/tg_db.log'
sn = 'ALDEBARAN' # servername
statusmsg = {}
name = "{}-DBopt".format(sn)
# Connect to test database
# ------------------------------------------------------------
try:
print("Connecting to DATABASE...")
def ValidityCheckDirectories(config={},statusmsg={}, debug=False):
"""
DESCRIPTION:
Check availability of paths for saving data products
"""
name0 = "{}-obligatory directories".format(config.get('logname','Dummy'))
statusmsg[name0] = 'all accessible'
successlist = [True,True]
vpath = config.get('variationpath')
qpath = config.get('quasidefinitivepath')
figpath = config.get('magfigurepath')
dipath = config.get('dipath')
dbcreds = config.get('dbcredentials')
if not isinstance(dbcreds,list):
dbcreds = [dbcreds]
try:
# Asuming dbpasswf is also good for mounting
dbcred = dbcreds[0] # only primary
dbpasswd = mpcred.lc(dbcred,'passwd')
except:
dbpasswd=''
def umount(path,pwd):
"""usage: umount("/srv/archive")"""
cmd = 'umount ' + path
print ("Sending umount command: {}".format(cmd))
echo = 'echo {}|sudo -S {}'.format(pwd,cmd)
subprocess.Popen(str(echo), shell=True, stdout=subprocess.PIPE)
print ("Done")
def mount(path,pwd):
"""usage: mount("/srv/archive")"""
cmd = 'mount ' + path
print ("Sending command: {}".format(cmd))
echo = 'echo {}|sudo -S {}'.format(pwd,cmd)
subprocess.Popen(str(echo), shell=True, stdout=subprocess.PIPE)
if not os.path.isdir(vpath) and not os.path.isdir(qpath) and not os.path.isdir(figpath):
print ("directory for products not accessible?")
statusmsg[name0] = 'products unavailable'
successlist[0] = False
# all other jobs cannot be performed
try:
print ("unmounting...")
umount("/srv/products",dbpasswd)
time.sleep(10)
print ("mounting products again...")
mount("-a",dbpasswd)
print ("success...")
successlist[0] = True
statusmsg[name0] = 'products unavailable - remounting successful'
except:
statusmsg[name0] = 'products unavailable - remounting failed'
if not os.path.isdir(dipath):
print ("archive not accessible?")
statusmsg[name0] = 'archive unavailable'
successlist[1] = False
try:
print ("unmounting...")
umount("/srv/archive",dbpasswd)
time.sleep(10)
print ("mounting archive again...")
mount("-a",dbpasswd)
print ("success...")
statusmsg[name0] = 'archive unavailable - remounting successful'
successlist[1] = True
except:
statusmsg[name0] = 'archive unavailable - remounting failed'
success = all([i for i in successlist])
return success, statusmsg
def process_data(station, sensortype, sensors, basepath, fileext, date,
plotvariables, **kwargs):
'''
DEFINITION:
STANDARD: Reads data, writes prelim data, plots & writes final data
OPTIONAL: Baseline & declination correct, multiply stream, upload,
produce plot of last 7 days.
NOTE: Files must be saved under proper data format:
--> /BASEPATH/TO/DATA/IAGA-ST-CODE/INSTRUMENT...
(e.g: /srv/archive/magnetism/tse/lemi...)
.../raw
.../plots
.../filtered
Must also have file with header information under BASEPATH/CODE/CODE_headers.txt.
PARAMETERS:
Variables:
- station: (str) IAGA-code for station
- sensortype: (str) 'combined' or 'normal'
combined = only for magnetic variometer + magnetometer
normal = all other data
- sensors: (str/list) Sensor name, 'ENV05_1_0001'
Note: for 'combined' this is a list. ['LEMI025_22_0001', 'POS1_N432_0001']
- basepath: (str) Path to where data is stored.
- fileext: (str/list) File extension of data file, e.g. 'bin', 'cdf', 'txt'
Note: for 'combined' this is a list. ['bin', 'cdf']
- date: (str) Date of data in format %Y-%m-%d / YYYY-MM-DD.
- plotvariables: (list) List of magpy keys to plot.
Kwargs:
- logger: (logging.Logger object) Logger for logging purposes.
- decl: (float) Will rotate data by this value, if given
- mult_factors: (dict) Will multiply corresponding stream keys with these factors
- baseline: (dict) Will baseline correct corresponding keys with these factors
- upload: (bool) If True, will upload files
- sevendayplot: (bool) If True, will produce plot of last 7 days
- prelim: (bool) If True, will save prelim file
RETURNS:
- True / False
EXAMPLE:
>>>
APPLICATION:
'''
decl = kwargs.get('decl')
mult_factors = kwargs.get('mult_factors')
baseline = kwargs.get('baseline')
upload = kwargs.get('upload')
prelim = kwargs.get('prelim')
sevendayplot = kwargs.get('sevendayplot')
logger = kwargs.get('logger')
if not logger:
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(' %s ' % station.upper())
header_data = {}
headersfile = os.path.join(basepath, station, '%s_headers.txt' % station)
headers = open(headersfile, 'r')
for line in headers:
hdata = line.split()
header_data[hdata[0].strip(':')] = hdata[1]
#--------------------------------------------------------------------
# 1. READ DATA, REMOVE OUTLIERS, FILTER
if sensortype == 'combined':
vario_sensor = sensors[0]
magn_sensor = sensors[1]
v_datafile = '%s_%s.%s' % (vario_sensor, date, fileext[0])
m_datafile = '%s_%s.%s' % (magn_sensor, date, fileext[1])
v_datapath = os.path.join(basepath, station, vario_sensor, 'raw',
v_datafile)
m_datapath = os.path.join(basepath, station, magn_sensor, 'raw',
m_datafile)
logger.info("Reading files %s and %s..." % (v_datafile, m_datafile))
v_stream = read(v_datapath)
m_stream = read(m_datapath)
#v_stream.remove_outlier()
#v_stream.remove_flagged()
#m_stream.remove_outlier()
#m_stream.remove_flagged()
v_stream = v_stream.filter()
m_stream = m_stream.filter()
stream = mergeStreams(v_stream, m_stream)
stream.header['col-f'] = 'F'
stream.header['unit-col-f'] = 'nT'
sensor = vario_sensor
title = '%s-%s' % (vario_sensor, magn_sensor)
elif sensortype == 'normal':
sensor = sensors
datafile = '%s_%s.%s' % (sensor, date, fileext)
logger.info("Reading file %s..." % datafile)
datapath = os.path.join(basepath, station, sensor, 'raw', datafile)
stream = read(datapath)
#stream.remove_outlier()
#stream.remove_flagged()
stream = stream.filter()
title = sensor
else:
logger.error(
"Wrong sensortype (%s). Options are 'combined' and 'normal'." %
sensortype)
for data_header in IAGA_headers:
stream.header[data_header] = header_data[data_header]
if sensor[:3].lower() == 'lem':
stream.header['DataType'] = 'Magnetic'
stream.header['DataComponents'] = 'x, y, z, F [nT]'
stream.header['DataDigitalSampling'] = '0.1s, 5s'
dx = 1000. * stream.header['DataCompensationX']
dy = 1000. * stream.header['DataCompensationY']
dz = 1000. * stream.header['DataCompensationZ']
stream.header['DataSensorOrientation'] = "%s, %s, %s" % (dx, dy, dz)
elif sensor[:3].lower() == 'env':
stream.header['DataType'] = 'Environmental'
stream.header[
'DataComponents'] = 'T (ambient) [C], RH [%], T (dewpoint) [C]'
stream.header['DataDigitalSampling'] = '1s'
stream._move_column(plotvariables[0], 'x')
stream._move_column(plotvariables[1], 'y')
stream._move_column(plotvariables[2], 'z')
plotvariables = ['x', 'y', 'z']
elif sensor[:3].lower() == 'cs1':
stream.header['DataType'] = 'Magnetic'
stream.header['DataComponents'] = 'F [nT]'
stream.header['DataDigitalSampling'] = '1s'
filenamebegins = '%s_0002' % (title)
#filenamebegins = '%s_%s_' % (station,title)
if prelim:
prelim_path = os.path.join(basepath, station, sensor, 'prelim')
stream.write(prelim_path,
filenamebegins=filenamebegins + '_',
format_type='IAGA')
logger.info("Preliminary data written to %s." % prelim_path)
#--------------------------------------------------------------------
# 2. (OPTIONAL) ROTATE, MULTIPLY, BASELINE CORRECT
# Steps for PRELIMINARY --> FINAL
if decl:
stream.rotation(alpha=decl)
if mult_factors:
stream.multiply(mult_factors)
if baseline:
stream.offset(baseline)
#--------------------------------------------------------------------
# 3. PLOT
sensorpadding = {'env': 0.5, 'pos': 10, 'lem': 5, 'cs1': 10}
plotname = '%s_%s.png' % (filenamebegins, date)
outfile = os.path.join(basepath, station, sensor, 'plots', plotname)
mp.plot(stream,
plotvariables,
plottitle='%s %s (%s)' % (station.upper(), title, date),
bgcolor='white',
confinex=True,
fullday=True,
outfile=outfile,
padding=sensorpadding[sensor[:3].lower()])
logger.info("Data plotted to %s." % outfile)
#--------------------------------------------------------------------
# 4. SAVE & WRITE STREAM TO MINUTE FILE
#filenamebegins = '%s_%s_' % (station,title)
finalpath = os.path.join(basepath, station, sensor, filenamebegins)
stream.write(finalpath,
filenamebegins=filenamebegins + '_',
format_type='IAGA')
logger.info("Final data written to %s." % finalpath)
#--------------------------------------------------------------------
# 5. UPLOAD (plot + filtered data)
cred = 'cobshomepage'
myproxy = mpcred.lc(cred, 'address')
login = mpcred.lc(cred, 'user')
passwd = mpcred.lc(cred, 'passwd')
#passwd = 'ku7tag8!haus' # TODO CHANGE THIS BACK
port = mpcred.lc(cred, 'port')
ftppath = 'cmsjoomla/images/stories/currentdata/'
upload = False
if upload:
try:
filtered_file = '%s_%s.txt' % (filenamebegins, date)
filtered_path = os.path.join(basepath, station, sensor,
filenamebegins, filtered_file)
logger.info("Uploading %s..." % filtered_path)
ftpdatatransfer(
localfile=filtered_path,
ftppath=ftppath, # TODO
myproxy=myproxy,
port=port,
login=login,
passwd=passwd,
raiseerror=True,
logfile=os.path.join(basepath, station,
'%s-transfer.log' % station))
except:
logger.error("Uploading failed.")
try:
plot_file = '%s_%s.png' % (filenamebegins, date)
plot_path = os.path.join(basepath, station, title, 'plots',
plotfile)
logger.info("Uploading %s..." % plot_path)
ftpdatatransfer(
localfile=plotpath,
ftppath=ftppath, # TODO
myproxy=myproxy,
port=port,
login=login,
passwd=passwd,
raiseerror=True,
logfile=os.path.join(basepath, station,
'%s-transfer.log' % station))
except:
logger.error("Uploading failed.")
#--------------------------------------------------------------------
# 6. CREATE 7-DAY PLOT (x, y, z, F) & UPLOAD
if sevendayplot:
today = datetime.utcnow()
date = datetime.strftime(today, "%Y-%m-%d")
datapath = os.path.join(basepath, station, sensor, filenamebegins, '*')
startdate = datetime.strptime(date, '%Y-%m-%d') - timedelta(days=7)
start = datetime.strftime(startdate, "%Y-%m-%d") + ' 00:00:00'
end = date + ' 00:00:00'
last7days = read(path_or_url=datapath, starttime=start, endtime=end)
plotname = 'TSE_last7days.png'
plotpath = os.path.join(basepath, station, '7dayplots', plotname)
diff = eval(last7days.header['DataSensorOrientation'])
last7days = last7days.offset(offsets={
'x': -float(diff[0]),
'y': -float(diff[1]),
'z': -float(diff[2])
})
last7days = last7days.calc_f()
fig = mp.plot(last7days, ['x', 'y', 'z', 'f'],
plottitle='%s Magnetic Data (%s - %s)' %
(station, start[:10], end[:10]),
bgcolor='white',
noshow=True,
padding=5)
axes = gcf().get_axes()
day = datetime.strptime(start, '%Y-%m-%d %H:%M:%S')
while day <= datetime.strptime(end, '%Y-%m-%d %H:%M:%S'):
if day.weekday() in [5, 6]: # Saturday or Sunday
t_start = day
t_end = day + timedelta(days=1)
for ax in axes:
ax.axvspan(t_start,
t_end,
facecolor='green',
alpha=0.3,
linewidth=0)
day += timedelta(days=1)
ax.get_xaxis().set_major_formatter(
matplotlib.dates.DateFormatter('%d.%b\n%H:%M'))
plt.savefig(plotpath, savedpi=80)
ftppath = 'zamg/images/graphs/magnetism/'
oldftppath = 'cmsjoomla/images/stories/currentdata/tse'
scptransfer(plotpath, '94.136.40.103:' + ftppath, passwd)
# Name of moon
#clientname = 'raspberrypi'
clientname = 'titan'
# IP of moon
#clientip = '192.168.178.47'
clientip = '138.22.188.182'
# Path of MARTAS directory on moon
martaspath = '/home/cobs/MARTAS'
# Provide Station code
stationid = 'MyHome'
# Select destination (file or db) - Files are saved in .../MARCOS/MoonsFiles/
dest = 'db'
# For Testing purposes - Print received data to screen:
printdata = True
# Please make sure that the db and scp connection data is stored within the credential file -otherwise provide this data directly
dbhost = mpcred.lc('mydb', 'host')
dbuser = mpcred.lc('mydb', 'user')
dbpasswd = mpcred.lc('mydb', 'passwd')
dbname = mpcred.lc('mydb', 'db')
scpuser = mpcred.lc('cobs', 'user')
scppasswd = mpcred.lc('cobs', 'passwd')
# You can add to the credential file by using:
# mpcred.cc('transfer','myshortcut',user='******',passwd='mypasswd',address='no-specific')
# and than read it by scpuser = mpcred.lc('myshortcut','myuser')
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# do necessary changes above
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
log.startLogging(sys.stdout)
sshcredlst = [scpuser, scppasswd]
# ----------------------------------------------------------
def main(argv):
shortcut = ''
path = ''
depth = 2
autofilter = 3
flagging = False
startdate = ''
archiveformat = 'PYCDF'
flaglist = []
skip = ''
samplingrateratio = 12 # 12 days * samplingperiod (sec) will be kept from today (e.g. 12 days of seconds data. 720 days of minute data.
try:
opts, args = getopt.getopt(argv, "hc:b:s:i:", [
"cred=",
"begin=",
"skip=",
"sr=",
])
except getopt.GetoptError:
print('deleteold.py -c <cred> -b <begin> -s <skip> -i <sr>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('-------------------------------------')
print('Description:')
print(
'-- deleteold.py gets data from a databank and deletes old data sets --'
)
print('Old database entries exceding a defined age')
print('are deleted.')
print('The databank size is automatically restricted ')
print(
'in dependency of the sampling rate of the input data. Only the last '
)
print(
'12 days of second data, the last 720 days of minute data and '
)
print(
'approximately 118 years of hourly data are kept. To modify these default'
)
print(
'settings please contact the developers (or learn python and')
print(
'edit the code - its simple and the MagPy cookbook will help you).'
)
print('-------------------------------------')
print('Usage:')
print('deleteold.py -c <cred> -b <begin> -s <skip> -i <sr> ')
print('-------------------------------------')
print('Options:')
print(
'-c (required) : provide the shortcut to the data bank credentials as defined by addcred.py'
)
print('-b : begin: not used so far')
print(
'-s : list sensor IDs to skip (comma separated list)'
)
print(
'-i : samplingrateratio for deleting old db entries - default is 12'
)
print(
' : deleting data older than samplingrate(sec)*12 days.'
)
print(
' : => i=12 : 1sec data older than 12 days is deleted in DB'
)
print(
' : 1min data older than 720 days is deleted in DB'
)
print(
' : => i=1 : 1sec data older than 1 day is deleted in DB'
)
print(
' : 1min data older than 60 days is deleted in DB'
)
print('-------------------------------------')
print('Example:')
print('every day cron job: python deleteold.py -c cobsdb')
print(
'creating archive of old db entries: python archive.py -c cobsdb -p /media/Samsung/Observatory/data/ -d 30 -b "2012-06-01" -g -i 100 -a 3'
)
sys.exit()
elif opt in ("-c", "--cred"):
cred = arg
elif opt in ("-b", "--begin"):
startdate = arg
elif opt in ("-s", "--skip"):
skip = arg
elif opt in ("-i", "--samplingrateratio"):
try:
samplingrateratio = int(arg)
except:
print("samplingrateratio needs to be an integer")
sys.exit()
if cred == '':
print(
'Specify a shortcut to the credential information by the -c option:'
)
print('-- check addcred.py -h for more options and requirements')
sys.exit()
print("Accessing data bank ...")
try:
db = mysql.connect(host=mpcred.lc(cred, 'host'),
user=mpcred.lc(cred, 'user'),
passwd=mpcred.lc(cred, 'passwd'),
db=mpcred.lc(cred, 'db'))
print("success")
except:
print("failure - check your credentials / databank")
sys.exit()
# Getting dates
datelist = []
if startdate == '':
current = datetime.utcnow() # make that a variable
else:
current = DataStream()._testtime(startdate)
newcurrent = current
for elem in range(depth):
datelist.append(datetime.strftime(newcurrent, "%Y-%m-%d"))
newcurrent = current - timedelta(days=elem + 1)
print("Dealing with time range:", datelist)
testdate = datetime.strftime(
(datetime.strptime(min(datelist), "%Y-%m-%d") - timedelta(days=1)),
"%Y-%m-%d")
# get a list with all datainfoids covering the selected time range
if startdate:
start = (datetime.strftime(DataStream()._testtime(startdate),
"%Y-%m-%d"))
sql = 'SELECT DataID FROM DATAINFO WHERE DataMaxTime > "' + start + '"'
else:
sql = 'SELECT DataID FROM DATAINFO WHERE DataMaxTime > "1900-01-01"'
print(sql)
# skip BLV measurements from cleanup
sql = sql + ' AND SensorID NOT LIKE "BLV_%"'
# skip Quakes from cleanup
sql = sql + ' AND SensorID NOT LIKE "QUAKES"'
if len(skip) > 0:
skipstr = ''
skiplst = skip.split(',')
for sensortoskip in skiplst:
skipstr += ' AND SensorID != "' + sensortoskip + '"'
sql = sql + skipstr
print sql
cursor = db.cursor()
try:
cursor.execute(sql)
except:
print("Error when reading database")
datainfoidlist = [elem[0] for elem in cursor.fetchall()]
print("Cleaning database contens of:", datainfoidlist)
for data in datainfoidlist:
print(" ---------------------------- ")
print(" ---------------------------- ")
print("Loading data files of", data)
print(" ---------------------------- ")
print(" ---------------------------- ")
print("Starting at: {}".format(datetime.utcnow()))
# Test of dataid table exists
try:
getline = True
amount = dbgetlines(db, data, 10000)
except:
print("Could not get lines from data file")
getline = False
if getline:
sr = amount.samplingrate()
delete = False
if amount.length()[0] > 0:
delete = True
print(" ---------- Deleting old data for {}".format(data))
else:
print(" ---------- Doing nothing for table {}".format(data))
#sql = 'DELETE FROM DATAINFO WHERE DataID = "{}"'.format(data)
#print (sql)
#cursor = db.cursor()
#cursor.execute(sql)
#db.commit()
#cursor.close()
if not isnan(sr) and delete and getline:
print("Now deleting old entries in database older than %s days" %
str(int(sr * samplingrateratio)))
dbdelete(db, data, samplingrateratio=samplingrateratio)
print(" -> ... success")
def main(argv):
creddb = '' # c
dipath = '' # a
variolist = '' # v
variodataidlist = '' # j
scalarlist = '' # s
scalardataidlist = '' # k
pierlist = '' # p
abstypelist = '' # y
azimuthlist = '' # z
archive = '' # w (e.g. /srv/archive)
identifier = 'BLV' # f
stationid = 'wic' # t
fallbackvariopath = '' # o
fallbackscalarpath = '' # l
begin = '1900-01-01' # b
end = datetime.strftime(datetime.utcnow(), "%Y-%m-%d") # e
expD = 3 # d
expI = 64 # i
compensation = False # m
rotation = False # q
dbadd = False # n
addBLVdb = False # n
flagging = False # g
createarchive = False # r
webdir = '/var/www/joomla/images/didaten/' # TODO add option
webuser = '******'
webgroup = 'www-data'
defaultuser = '******'
defaultgroup = 'cobs'
debug = False
flaglist = []
keepremote = False
getremote = False
remotecred = ''
remotepath = ''
variopath = '' #
scalarpath = '' #
try:
opts, args = getopt.getopt(
argv, "hc:a:v:j:s:k:o:mql:b:e:t:z:d:i:p:y:w:f:ngrx:u:D", [
"cred=", "dipath=", "variolist=", "variodataidlist=",
"scalarlist=", "scalardataidlist=", "variopath=",
"compensation=", "rotation=", "scalarpath=", "begin=", "end=",
"stationid=", "pierlist=", "abstypelist=", "azimuthlist=",
"expD=", "expI=", "write=", "identifier=", "add2DB=", "flag=",
"createarchive=", "webdir=", "keepremote", "debug="
])
except getopt.GetoptError:
print(
'di.py -c <creddb> -a <dipath> -v <variolist> -j <variodataidlist> -s <scalarlist> -o <variopath> -m <compensation> -q <rotation> -l <scalarpath> -b <startdate> -e <enddate> -t <stationid> -p <pierlist> -z <azimuthlist> -y <abstypelist> -d <expectedD> -i <expectedI> -w <writepath> -f<identifier> -n <add2DB> -g <flag> -r <createarchive> -x <webdir> -u <user> --keepremote'
)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('-------------------------------------')
print('Description:')
print(
'dianalysis.py reads DI measurements and calculates DI values.'
)
print('Provide variometer and scalar data for correction.')
print('Returns di values, f and collimation angles.')
print(
'A number of additional option allow for archiving, validity tests,'
)
print(
'and output redirection. If variometer data is provided, base values'
)
print('are calculated. ')
print('')
print('-------------------------------------')
print('Usage:')
print(
'di.py -c <creddb> -a <dipath> -v <variolist> -j <variodataidlist> -s <scalarlist> -o <variopath> -l <scalarpath> -m <compensation> -q <rotation> -b <startdate> -e <enddate> -t <stationid> -p <pierlist> -z <azimuthlist> -y <abstypelist> -d <expectedD> -i <expectedI> -w <writepath> -n <add2DB> -g <flag> -r <createarchive> -x <webdir> -u <user> --keepremote'
)
print('-------------------------------------')
print('Options:')
print(
'-c : provide the shortcut to the data bank credentials'
)
print(
'-a (required) : path to DI data - can be either a real path to a local directory'
)
print(
' or a credential shortcut for a remote connection '
)
print(
'-w (required) : archivepath for writing results and eventually accessing data.'
)
print(' e.g. /srv/archive. or /tmp')
print(
' below this folder the following structure will be implemented:'
)
print(' -/srv/archive/"StationID"/DI/analyse : ')
print(
' folder with raw data to be analyzed'
)
print(' -/srv/archive/"StationID"/DI/data : ')
print(
' folder with calculated di results'
)
print(
' Name will be set to "BLV_" + variometerID + pier.'
)
print(
' This plain text file can be opend and'
)
print(
' analyzed with the MagPy stream package.'
)
print(' -/srv/archive/"StationID"/DI/raw : ')
print(
' archivefolder with successfully analyzed raw data'
)
print(
'-f : idetifier for BLV data (in database and filename) - default is BLV'
)
print(
'-v : variolist - comma separated list of variometer ids'
)
print(
'-j : variodataidlist - specify the dataids to be used fro each vario'
)
print(' Default: 0002 for each variometer sensor')
print('-o : path to variometer data')
print(
'-m (no input) : apply compensation field values to variometer data'
)
print('-q (no input) : apply rotation to variometer data')
print('-s : scalarpath - path to scalar data')
print(
'-k : scalardataidlist - specify the dataids to be used fro each scalar'
)
print(' Default: 0002 for each scalar sensor')
print(
'-b : startdate - begin of analysis (not yet active)'
)
print(
'-e : enddate - default is today (not yet active)')
print(
'-t (required) : ID of the station i.e. the Observatory code (required if'
)
print(' not in meta data of DI measurements)')
print(
'-z : list of astronomic azimuths of the mark from the measurement pier'
)
print(
' : Azimuthlist needs either to be empty or to have the same order '
)
print(' and length of the pierlist.')
print(
' use "False" if the specific value should be taken from the'
)
print(' originalfile/database')
print(' e.g. -p D3,H6 -z False,156.678')
print(
'-y : comma separated list of absolute data types for each pier'
)
print(' : "di" for standard theodolite or "autodif" ')
print(' : e.g. -p D3,H6 -m di,autodif ')
print('-d : expected declination')
print('-i : expected inclination')
print(
'-p (required) : name/number of the pier, comma separated list'
)
print('-n (no input) : add di and basevalues to data base')
print(
'-g (no input) : read flaglist from DB if db is opened and add flags'
)
print(
'-r : move successfully analyzed files to raw archive'
)
print('-x : directory to copy non-analyzed files to.')
print(
' : can be a www directory at which PHP-scripts are used to edit data.'
)
print('-u : define user for which jobs are performed')
print(' : e.g. cobs:cobsgroup')
print(
'--keepremote : Don t delete remote files after dowloading them'
)
print('-------------------------------------')
print('Examples:')
print('1. Running on MARCOS servers:')
print(
'python di.py -c wic -a cobshome,cobenzlabs -v "FGE_S0252_0001"'
)
print(' -s "POS1_N432_0001" -j 0002 -b 2014-01-01')
print(
' -w /media/DAE2-4808/archive -d 3 -i 64 -t wic -p H1,A7,A2,A16'
)
print(
' -y di,di,di,autodif -z False,179.8978,180.1391,267.3982'
)
print('2. Running it with manually provided data links:')
print(
'python di.py -c wic -a /media/DAE2-4808/archive/WIC/DI/analyze'
)
print(
' -v "FGE_S0252_0001" -s "POS1_N432_0001" -j 0002 -b 2014-02-01'
)
print(
' -e 2014-05-01 -w /media/DAE2-4808/archive -d 3 -i 64 -t wic'
)
print(' -p H1,A7,A2,A16 -y di,di,di,autodif -r')
print(' -z False,179.8978,180.1391,267.3982 -u user:group')
print('python di.py -c cobs -a cobshomepage,cobenzlabs ')
print(
' -v DIDD_3121331_0002,LEMI025_1_0002 -s DIDD_3121331_0002'
)
print(' -j 0001,0001 -b 2014-10-01 -e 2014-10-07')
print(' -w /srv/archive -d 3 -i 64 -t wik -p D -n -r')
sys.exit()
elif opt in ("-c", "--creddb"):
creddb = arg
elif opt in ("-a", "--dipath"):
dipath = arg
elif opt in ("-w", "--archive"):
archive = arg
elif opt in ("-f", "--identifier"):
identifier = arg
elif opt in ("-v", "--variolist"):
variolist = arg.split(',')
elif opt in ("-j", "--variodataidlist"):
variodataidlist = arg.split(',')
elif opt in ("-s", "--scalarlist"):
scalarlist = arg.split(',')
elif opt in ("-k", "--scalardataidlist"):
scalardataidlist = arg.split(',')
elif opt in ("-o", "--variopath"):
fallbackvariopath = arg
elif opt in ("-m", "--compensation"):
compensation = True
elif opt in ("-q", "--rotation"):
rotation = True
elif opt in ("-l", "--scalarpath"):
fallbackscalarpath = arg
elif opt in ("-b", "--begin"):
begin = arg
elif opt in ("-e", "--end"):
end = arg
elif opt in ("-t", "--stationid"):
stationid = arg
elif opt in ("-p", "--pierlist"):
pierlist = arg.split(',')
elif opt in ("-z", "--azimuthlist"):
azimuthlist = arg.split(',')
elif opt in ("-y", "--abstypelist"):
abstypelist = arg.split(',')
elif opt in ("-x", "--webdir"):
webdir = arg
elif opt in ("-u", "--user"):
user = arg.split(':')
if len(user) > 1:
defaultuser = user[0]
defaultgroup = user[1]
elif opt in ("-d", "--expectedD"):
try:
expD = float(arg)
except:
print("expected declination needs to be a float")
sys.exit()
elif opt in ("-i", "--expectedI"):
try:
expI = float(arg)
except:
print("expected inclination needs to be a float")
sys.exit()
elif opt in ("-n", "--add2db"):
dbadd = True
addBLVdb = True
elif opt in ("-g", "--flag"):
flagging = True
elif opt in ("--keepremote"):
keepremote = True
elif opt in ("-r", "--createarchive"):
createarchive = True
elif opt in ("-D", "--debug"):
debug = True
print("-------------------------------------")
print("Starting di analysis ... MARTAS version {}".format(__version__))
print("-------------------------------------")
if dipath == '':
print(' Specify the path to the DI data: -a /path/to/my/data !')
print(' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if archive == '':
print(
' Specify an Archive path for writing results: -w /path/to/my/archive !'
)
print(' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if variolist == '':
variolist = []
if scalarlist == '':
scalarlist = []
if azimuthlist == '':
azimuthlist = []
if abstypelist == '' or len(abstypelist) == 0:
abstypelist = ['di' for elem in pierlist]
if variodataidlist == '':
variodataidlist = []
if scalardataidlist == '':
scalardataidlist = []
if len(variodataidlist) == 0:
variodataidlist = ['0002' for elem in variolist]
else:
if not len(variolist) == len(variodataidlist):
print(
' You need to specify a specific DataID for each variometer: e.g. -j 0002,0001'
)
print(
' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if len(scalardataidlist) == 0:
scalardataidlist = ['0002' for elem in scalarlist]
else:
if not len(scalarlist) == len(scalardataidlist):
print(
' You need to specify a specific DataID for each variometer: e.g. -j 0002,0001'
)
print(
' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if not len(abstypelist) == 0:
if not len(abstypelist) == len(pierlist):
print(
' Abstypelist needs to have the same order and length of the pierlist'
)
print(
' -- check dianalysis.py -h for more options and requirements')
sys.exit()
try:
test = datetime.strptime(begin, "%Y-%m-%d")
print(test)
except:
print(' Date format for begin seems to be wrong: -b 2013-11-22')
print(' -- check dianalysis.py -h for more options and requirements')
sys.exit()
try:
datetime.strptime(end, "%Y-%m-%d")
except:
print(' Date format for end seems to be wrong: -e 2013-11-22')
print(' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if pierlist == []:
print(
' Specify a list of the measurement piers containing at list one element: -p [Pier2]'
)
print(' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if not len(azimuthlist) == 0:
if not len(azimuthlist) == len(pierlist):
print(
' Azimuthlist needs to have the same order and length of the pierlist'
)
print(
' -- check dianalysis.py -h for more options and requirements')
sys.exit()
if stationid == '':
print(' Specify a station name e.g. your observatory code')
print(' -- check dianalysis.py -h for more options and requirements')
sys.exit()
else:
stationid = stationid.upper()
if not creddb == '':
print(" Accessing data bank ...")
try:
db = mysql.connect(host=mpcred.lc(creddb, 'host'),
user=mpcred.lc(creddb, 'user'),
passwd=mpcred.lc(creddb, 'passwd'),
db=mpcred.lc(creddb, 'db'))
print(" ... success")
except:
print(" ... failure - check your credentials")
sys.exit()
else:
db = False
if not fallbackvariopath == '':
if not fallbackvariopath.endswith('*'):
fallbackvariopath = os.path.join(fallbackvariopath, '*')
variopath = fallbackvariopath
if not fallbackscalarpath == '':
if not fallbackscalarpath.endswith('*'):
fallbackscalarpath = os.path.join(fallbackscalarpath, '*')
scalarpath = fallbackscalarpath
if variolist == []:
if fallbackvariopath == '':
print(
' !! You have not provided any variometer information at all')
if scalarlist == []:
if fallbackscalarpath == '':
print(' You have not provided any independent scalar information')
print(
' -> we asume this data is provided along with the DI files')
# -----------------------------------------------------
# a) Basic information
# -----------------------------------------------------
if debug:
print(" -------------------------------------")
print(" Archive", archive)
print(" Variolist", variolist)
print(" Abstypelist", abstypelist)
print(" Dipath", dipath)
# -----------------------------------------------------
# b) Getting new raw data from the input server
# -----------------------------------------------------
print(" -------------------------------------")
print(" Identifying DI data source")
if not os.path.exists(dipath):
print(" Checking given DI path for credential information ...")
try:
credlist = mpcred.sc()
credshort = [elem[0] for elem in credlist]
print(" ... found credentials")
except:
print(
" dipath {} not existing - credentials not accessible - aborting"
.format(dipath))
sys.exit()
try:
dic = dipath.split(',')
print(dic)
print(len(dic))
if len(dic) == 2:
remotecred = dic[0]
remotepath = dic[1]
print(
" Using credentials {} to get DI data from the remote path {}"
.format(dic[0], dic[1]))
elif len(dic) == 1:
remotecred = dic[0]
remotepath = ''
print(" Using credentials {} to get DI data".format(dic[0]))
else:
print(
" -> could not interprete dipath in terms of credential information"
)
sys.exit()
if remotecred in credshort:
getremote = True
else:
print(
" -> dipath %s not existing - credentials not existing - aborting"
% dipath)
except:
print(
" -> dipath %s not existing - credentials not existing - aborting"
% dipath)
sys.exit()
if getremote == False:
sys.exit()
else:
print(" Found directory at specified dipath location")
# Getting data from the webdir (eventually edited and corrected)
if createarchive and not webdir == '':
print(" -------------------------------------")
print(" Createarchive and webdir selected ...")
dipath = os.path.join(archive, stationid, 'DI', 'analyze')
for pier in pierlist:
diid = pier + '_' + stationid + '.txt'
for infile in iglob(os.path.join(webdir, '*' + diid)):
# Testing whether file exists:
if os.path.exists(
os.path.join(dipath,
os.path.split(infile)[1])):
print(" Deleting:",
os.path.join(dipath,
os.path.split(infile)[1]))
os.remove(os.path.join(dipath, os.path.split(infile)[1]))
print(" Retrieving from webdir: ", infile)
shutil.copy(infile, dipath)
# Setting permission to defaultuser even if started the job
uid = pwd.getpwnam(defaultuser)[2]
gid = grp.getgrnam(defaultgroup)[2]
os.chown(os.path.join(dipath,
os.path.split(infile)[1]), uid, gid)
# Deleting file from web dir
try:
os.remove(os.path.join(webdir, os.path.split(infile)[1]))
except:
print(" !! No permissions to modify webdirectory")
pass
# copy all files from web directory to the analysis folder
if getremote:
delete = True
if keepremote:
delete = False
print(" -------------------------------------")
print(
" Getting remote data - deleting downloaded data from remote source set to {}"
.format(delete))
dipath = os.path.join(archive, stationid, 'DI', 'analyze')
for pier in pierlist:
if not os.path.exists(dipath):
os.makedirs(dipath)
diid = pier + '_' + stationid + '.txt'
try:
port = mpcred.lc(remotecred, 'port')
except:
port = 21
ftpget(mpcred.lc(remotecred, 'address'),
mpcred.lc(remotecred, 'user'),
mpcred.lc(remotecred, 'passwd'),
remotepath,
os.path.join(archive, stationid, 'DI', 'analyze'),
diid,
port=port,
delete=delete)
if debug:
print(" -------------------------------------")
print(
" DI data defined and collected - now starting the analysis for variometer: {}"
.format(variolist))
print(" ")
# -----------------------------------------------------
# c) analyze all files in the local analysis directory and put successfully analyzed data to raw
# -----------------------------------------------------
for pier in pierlist:
print("######################################################")
print("Starting analysis for pier ", pier)
print("######################################################")
abspath = dipath
diid = pier + '_' + stationid + '.txt'
for vario in variolist:
dataid = variodataidlist[variolist.index(vario)]
if os.path.exists(
os.path.join(archive, stationid, vario,
vario + '_' + dataid)):
variopath = os.path.join(archive, stationid, vario,
vario + '_' + dataid, vario + '*')
else:
variopath = vario
if not os.path.exists(variopath):
print(
" -> No variometerdata found in the specified paths/IDs - using dummy path"
)
variopath = '/tmp/*'
print(" -> Using Variometerdata at:", variopath)
for scalar in scalarlist:
# Define paths for variometer and scalar data
scalarid = scalardataidlist[scalarlist.index(scalar)]
if os.path.exists(
os.path.join(archive, stationid, scalar,
scalar + '_' + scalarid)):
scalarpath = os.path.join(archive, stationid, scalar,
scalar + '_' + scalarid,
scalar + '*')
else:
scalarpath = scalar
if not os.path.exists(scalarpath):
print(
" -> No scalar data found in the specified paths/IDs - using dummy path"
)
scalarpath = '/tmp/*'
print(" -> Using Scalar data at:", scalarpath)
# ALPHA and delta needs to be provided with the database
print(" -------------------------------------")
print(
" Extracting delta and rotation parameters ... should not be necessary as this is be done by absoluteAnalysis provided a database is connected"
)
deltaF = 0.0
alpha = 0.0
beta = 0.0
"""
if db:
print(" ")
alpha = dbgetfloat(db, 'DATAINFO', vario, 'DataSensorAzimuth')
if not isNumber(alpha):
alpha = 0.0
beta = dbgetfloat(db, 'DATAINFO', vario, 'DataSensorTilt')
if not isNumber(beta):
beta = 0.0
deltaF = dbgetfloat(db, 'DATAINFO', scalar, 'DataDeltaF')
if not isNumber(deltaF):
deltaF = 0.0
else:
# eventually add an input option
# load a scalar file from path and get delta F from header
try:
scal = read(scalarpath,starttime=begin,endtime=begin)
try:
scal = applyDeltas(db,scal)
deltaF = 0.0
except:
deltaF = scal.header['DataDeltaF']
except:
deltaF = 0.0
try:
var = read(variopath,starttime=begin,endtime=begin)
try:
var = applyDeltas(db,var)
except:
pass
# TODO this is wrong -> but clarify whether a correction is necessary at all
alpha = var.header['DataSensorAzimuth']
beta = var.header['DataSensorTilt']
except:
alpha = 0.0
beta = 0.0
print("using alpha, beta, deltaF:", alpha, beta, deltaF)
"""
print(" -------------------------------------")
print(
" Extracting azimuth data ... should be contained in DI files, can be provided as option, is contained in PIERS table of DB"
)
# Azimuths are usually contained in the DI files
## Eventually overriding azimuths in DI files
if len(azimuthlist) > 0:
azimuth = azimuthlist[pierlist.index(pier)]
if azimuth == 'False' or azimuth == 'false':
azimuth = False
else:
azimuth = False
if azimuth:
print(
" -> Overriding (eventual) DI files data with an azimuth of {} deg"
.format(azimuth))
else:
print(" -> Using azimuth from DI file")
if len(abstypelist) > 0:
abstype = abstypelist[pierlist.index(pier)]
if abstype == 'False' or abstype == 'false':
abstype = False
else:
abstype = False
if abstype:
print(" -> Selected type of absolute measurements is {}".
format(abstype))
else:
print(" -> Absolute measurement type taken from DI file")
# TODO ... Get azimuth data from PIERS table
if db:
print(
" Checking azimuth in PIERS table of the database ...")
val = dbselect(db, 'AzimuthDictionary', 'PIERS',
'PierID like "{}"'.format(pier))[0]
print("Found ", val)
print(" -------------------------------------")
movetoarchive = False
if createarchive and variolist.index(
vario) == len(variolist) - 1 and scalarlist.index(
scalar) == len(scalarlist) - 1:
print(
" Running analysis - and moving successfully analyzed files to raw directory"
)
movetoarchive = os.path.join(archive, stationid, 'DI',
'raw')
else:
print(
" Running analysis - and keeping files in analyze directory"
)
absstream = absoluteAnalysis(abspath,
variopath,
scalarpath,
expD=expD,
expI=expI,
diid=diid,
stationid=stationid,
abstype=abstype,
azimuth=azimuth,
pier=pier,
alpha=alpha,
deltaF=deltaF,
starttime=begin,
endtime=end,
db=db,
dbadd=dbadd,
compensation=compensation,
magrotation=rotation,
movetoarchive=movetoarchive,
deltaD=0.0000000001,
deltaI=0.0000000001)
print(" -> Done")
# -----------------------------------------------------
# d) write data to a file and sort it, write it again
# (workaround to get sorting correctly)
# -----------------------------------------------------
print(" -------------------------------------")
if absstream and absstream.length()[0] > 0:
print(" Writing {} data line(s) ...".format(
absstream.length()[0]))
absstream.write(os.path.join(archive, stationid, 'DI',
'data'),
coverage='all',
mode='replace',
filenamebegins=identifier + '_' + vario +
'_' + scalar + '_' + pier)
try:
# Reload all data, delete old file and write again to get correct ordering
newabsstream = read(
os.path.join(
archive, stationid, 'DI', 'data', identifier +
'_' + vario + '_' + scalar + '_' + pier + '*'))
os.remove(
os.path.join(
archive, stationid, 'DI', 'data',
identifier + '_' + vario + '_' + scalar + '_' +
pier + '.txt')) # delete file from hd
newabsstream.write(os.path.join(
archive, stationid, 'DI', 'data'),
coverage='all',
mode='replace',
filenamebegins=identifier + '_' +
vario + '_' + scalar + '_' + pier)
except:
print(" Stream apparently not existing...")
print(" -> Done")
if addBLVdb:
# SensorID necessary....
print(" Adding data to the data bank ... ")
#newabsstream.header["SensorID"] = vario
writeDB(db,
absstream,
tablename=identifier + '_' + vario + '_' +
scalar + '_' + pier)
#stream2db(db,newabsstream,mode='force',tablename=identifier+'_'+vario+'_'+scalar+'_'+pier)
print(" -> Done")
# -----------------------------------------------------
# f) get flags and apply them to data
# -----------------------------------------------------
if db and flagging and addBLVdb:
newabsstream = readDB(
db, identifier + '_' + vario + '_' + scalar + '_' +
pier)
flaglist = db2flaglist(
db, identifier + '_' + vario + '_' + scalar + '_' +
pier)
elif addBLVdb:
newabsstream = readDB(
db, identifier + '_' + vario + '_' + scalar + '_' +
pier)
flaglist = []
if len(flaglist) > 0:
flabsstream = newabsstream.flag(flaglist)
#for i in range(len(flaglist)):
# flabsstream = newabsstream.flag_stream(flaglist[i][2],flaglist[i][3],flaglist[i][4],flaglist[i][0],flaglist[i][1])
flabsstream.write(os.path.join(archive, stationid,
'DI', 'data'),
coverage='all',
filenamebegins=identifier + '_' +
vario + '_' + scalar + '_' + pier)
pltabsstream = flabsstream.remove_flagged()
# -----------------------------------------------------
# h) fit baseline and plot
# -----------------------------------------------------
try:
#pltabsstream = read(os.path.join(archive,stationid,'DI','data',identifier+'_'+vario+'_'+scalar+'_'+pier+'*'))
pltabsstream.trim(starttime=datetime.utcnow() -
timedelta(days=380))
# fit baseline using the parameters defined in db (if parameters not available then skip fitting)
#absstream = absstream.fit(['dx','dy','dz'],poly,4)
savename = identifier + '_' + vario + '_' + scalar + '_' + pier + '.png'
#absstream = absstream.extract('f',98999,'<')
mp.plot(pltabsstream, ['dx', 'dy', 'dz'],
symbollist=['o', 'o', 'o'],
plottitle=vario + '_' + scalar + '_' + pier,
outfile=os.path.join(archive, stationid, 'DI',
'graphs', savename))
#absstream.plot(['dx','dy','dz'],symbollist=['o','o','o'],plottitle=vario+'_'+scalar+'_'+pier,outfile=os.path.join(archive,stationid,'DI','graphs',savename))
except:
pass
# -----------------------------------------------------
# j) move files from analyze folder to web folder
# -----------------------------------------------------
# move only if createarchive is selected
if createarchive:
print(" -------------------------------------")
print(
" Dont mind the error message - works only if su at cron is running this job"
)
filelst = []
for infile in iglob(
os.path.join(archive, stationid, 'DI', 'analyze', '*.txt')):
print("Processing ", infile)
filelst.append(infile)
destination = '/var/www/joomla/images/didaten/'
infilename = os.path.split(infile)
print(infilename)
try:
shutil.copy(infile, destination)
#perform changes to privs
if not webuser == '':
uid = pwd.getpwnam(webuser)[2]
gid = grp.getgrnam(webgroup)[2]
os.chown(os.path.join(destination, infilename[1]), uid,
gid)
except:
print("Webdir not accessible - finishing")
pass
print("----------------------------------------------------------------")
print("di.py app finished")
print("----------------------------------------------------------------")
print("SUCCESS")
# lastfiles looks like: {'/path/to/my/file81698.txt' : '2019-01-01T12:33:12', ...}
if not lastfiles == {}:
print ("write memory")
pass
sourcepath = workdictionary.get(key).get('path')
starttime = workdictionary.get(key).get('starttime')
endtime = workdictionary.get(key).get('endtime')
newfiledict, alldic = getchangedfiles(sourcepath, lastfiles, starttime, endtime)
print ("Found new: {} and all {}".format(newfiledict, alldic))
for dest in workdictionary.get(key).get('destinations'):
print (" -> Destination: {}".format(dest))
address=mpcred.lc(dest,'address')
user=mpcred.lc(dest,'user')
passwd=mpcred.lc(dest,'passwd')
port=mpcred.lc(dest,'port')
destdict = workdictionary.get(key).get('destinations')[dest]
#print (destdict)
if address and user and newfiledict:
for nfile in newfiledict:
print (" -> Uploading {} to dest {}".format(nfile, dest))
success = uploaddata(nfile, destdict.get('path'), destdict.get('type'), address, user, passwd, port, logfile=destdict.get('logfile','stdout'))
print (" -> Success", success)
if not success:
#remove nfile from alldic
# thus it will be retried again next time
print (" !---> upload of {} not successful: keeping it in todo list".format(nfile))
del alldic[nfile]
# Name of moon
#clientname = 'raspberrypi'
clientname = 'titan'
# IP of moon
#clientip = '192.168.178.47'
clientip = '138.22.188.182'
# Path of MARTAS directory on moon
martaspath = '/home/cobs/MARTAS'
# Provide Station code
stationid = 'MyHome'
# Select destination (file or db) - Files are saved in .../MARCOS/MoonsFiles/
dest = 'db'
# For Testing purposes - Print received data to screen:
printdata = True
# Please make sure that the db and scp connection data is stored within the credential file -otherwise provide this data directly
dbhost = mpcred.lc('mydb','host')
dbuser = mpcred.lc('mydb','user')
dbpasswd = mpcred.lc('mydb','passwd')
dbname = mpcred.lc('mydb','db')
scpuser = mpcred.lc('cobs','user')
scppasswd = mpcred.lc('cobs','passwd')
# You can add to the credential file by using:
# mpcred.cc('transfer','myshortcut',user='******',passwd='mypasswd',address='no-specific')
# and than read it by scpuser = mpcred.lc('myshortcut','myuser')
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# do necessary changes above
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
log.startLogging(sys.stdout)
sshcredlst = [scpuser,scppasswd]
def main(argv):
#broker = '192.168.178.75'
broker = 'localhost' # default
#broker = '192.168.178.84'
#broker = '192.168.0.14'
port = 1883
timeout=60
user = ''
password = ''
global destination
destination='stdout'
global location
location='/tmp'
global credentials
credentials=''
global offset
offset = ''
global dbcred
dbcred=''
global stationid
stationid = 'wic'
global source
source='mqtt' # projected sources: mqtt (default), wamp, mysql, postgres, etc
global qos
qos=0
global debug
debug = False
global output
global headstream
headstream = {}
#global verifiedlocation
#verifiedlocation = False
global dictcheck
dictcheck = False
usagestring = 'collector.py -b <broker> -p <port> -t <timeout> -o <topic> -d <destination> -l <location> -c <credentials> -r <dbcred> -q <qos> -u <user> -P <password> -s <source> -f <offset>'
try:
opts, args = getopt.getopt(argv,"hb:p:t:o:d:l:c:r:q:u:P:s:f:U",["broker=","port=","timeout=","topic=","destination=","location=","credentials=","dbcred=","qos=","debug=","user="******"password="******"source=","offset="])
except getopt.GetoptError:
print('Check your options:')
print(usagestring)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('------------------------------------------------------')
print('Usage:')
print(usagestring)
print('------------------------------------------------------')
print('Options:')
print('-h help')
print('-b set broker address: default = localhost')
print('-p set port - default is 1883')
print('-t set timeout - default is 60')
print('-o set base topic - for MARTAS this corresponds')
print(' to the station ID (e.g. wic)')
print('-d set destination - std.out, db, file')
print(' default is std.out')
print('-l set location depending on destination')
print(' if d="file": provide path')
print(' if d="db": provide db credentials')
print(' if d="std.out": not used')
print('-c set mqtt communication credential keyword')
print('-q set mqtt quality of service (default = 0)')
print('-r set db credential keyword')
print('-s source protocol of data: default is mqtt')
print(' other options:')
print(' -s wamp not yet implemented.')
print('-f offset values. Can either be "db" for')
print(' applying delta values from db or a string')
print(' of the following format (key:value):')
print(' -f "t1:3.234,x:45674.2"')
print(' other options:')
print('------------------------------------------------------')
print('Examples:')
print('1. Basic')
print(' python collector.py -b "192.168.0.100" -o wic')
print('2. Writing to file in directory "/my/path/"')
print(' python collector.py -b "192.168.0.100" -d file -l "/my/path" -o wic')
print('3. Writing to file and stdout')
print(' python collector.py -b "192.168.0.100" -d file,stdout -l "/tmp" -o wic')
print('4. Writing to db')
print(' python collector.py -b "192.168.0.100" -d db -r mydb -o wic')
print(' python collector.py -d "db,file" -r testdb')
sys.exit()
elif opt in ("-b", "--broker"):
broker = arg
elif opt in ("-p", "--port"):
try:
port = int(arg)
except:
port = arg
elif opt in ("-t", "--timeout"):
try:
timeout = int(arg)
except:
timeout = arg
elif opt in ("-o", "--topic"):
stationid = arg
elif opt in ("-s", "--source"):
source = arg
elif opt in ("-d", "--destination"):
destination = arg
elif opt in ("-l", "--location"):
location = arg
elif opt in ("-c", "--credentials"):
credentials = arg
elif opt in ("-r", "--dbcred"):
dbcred = arg
elif opt in ("-q", "--qos"):
try:
qos = int(arg)
except:
qos = 0
elif opt in ("-u", "--user"):
user = arg
elif opt in ("-P", "--password"):
password = arg
elif opt in ("-f", "--offset"):
offset = arg
elif opt in ("-U", "--debug"):
debug = True
if not qos in [0,1,2]:
qos = 0
if 'stringio' in destination:
output = StringIO.StringIO()
if 'file' in destination:
if location in [None,''] and not os.path.exists(location):
print ('destination "file" requires a valid path provided as location')
print (' ... aborting ...')
sys.exit()
if 'websocket' == destination:
if ws_available:
wsThr = threading.Thread(target=wsThread)
print('starting websocket on port 5000...')
wsThr.start()
else:
print("use pip install websocket-server")
sys.exit()
if 'db' in destination:
if dbcred in [None,'']:
print ('destination "db" requires database credentials')
print ('to create them use method "addcred"')
print ('to provide use option -r like -r mydb')
sys.exit()
else:
try:
global db
if debug:
print ("Connecting database {} at host {} with user {}".format(mpcred.lc(dbcred,'db'),mpcred.lc(dbcred,'host'),mpcred.lc(dbcred,'user')))
db = mysql.connect(host=mpcred.lc(dbcred,'host'),user=mpcred.lc(dbcred,'user'),passwd=mpcred.lc(dbcred,'passwd'),db=mpcred.lc(dbcred,'db'))
except:
print ('database coul not be connected')
print (' ... aborting ...')
sys.exit()
if debug:
print ("Option u: debug mode switched on ...")
print ("------------------------------------")
print ("Destination", destination, location)
if source == 'mqtt':
client = mqtt.Client()
# Authentication part
if not user in ['',None,'None','-']:
#client.tls_set(tlspath) # check http://www.steves-internet-guide.com/mosquitto-tls/
client.username_pw_set(user, password=password) # defined on broker by mosquitto_passwd -c passwordfile user
client.on_connect = on_connect
# on message needs: stationid, destination, location
client.on_message = on_message
client.connect(broker, port, timeout)
client.loop_forever()
elif source == 'wamp':
print ("Not yet supported! -> check autobahn import, crossbario")
else:
print ("Additional protocols can be added in future:")
def main(argv):
broker = 'localhost' # default
port = 1883
timeout = 60
altbroker = ''
user = ''
password = ''
logging = 'sys.stdout'
global destination
destination = 'stdout'
global location
location = '/tmp'
global credentials
credentials = ''
global offset
offset = ''
global dbcred
dbcred = ''
global stationid
stationid = 'wic'
global stid
stid = stationid
global webpath
webpath = './web'
global webport
webport = 8080
global instrument
instrument = ''
global revision
revision = 'fix'
global telegramconf
telegramconf = ''
global addlib
addlib = []
global source
source = 'mqtt' # projected sources: mqtt (default), wamp, mysql, postgres, etc
global debug
debug = False
global output
global headstream
headstream = {}
global topic_identifiers
topic_identifiers = {}
global class_reference
class_reference = {}
#global verifiedlocation
#verifiedlocation = False
global dictcheck
dictcheck = False
global socketport
global number
number = 1
global qos
qos = 0
global blacklist
blacklist = []
usagestring = 'collector.py -b <broker> -p <port> -t <timeout> -o <topic> -i <instrument> -d <destination> -v <revision> -l <location> -c <credentials> -r <dbcred> -q <qos> -u <user> -P <password> -s <source> -f <offset> -m <marcos> -n <number> -e <telegramconf> -a <addlib>'
try:
opts, args = getopt.getopt(
argv, "hb:p:t:o:i:d:vl:c:r:q:u:P:s:f:m:n:e:a:U", [
"broker=", "port=", "timeout=", "topic=", "instrument=",
"destination=", "revision=", "location=", "credentials=",
"dbcred=", "qos=", "debug=", "user="******"password="******"source=",
"offset=", "marcos=", "number=", "telegramconf=", "addlib="
])
except getopt.GetoptError:
print('Check your options:')
print(usagestring)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('------------------------------------------------------')
print('Usage:')
print(usagestring)
print('------------------------------------------------------')
print('Options:')
print('-h help')
print(
'-b set broker address: default = localhost'
)
print('-p set port - default is 1883')
print('-t set timeout - default is 60')
print(
'-o set base topic - for MARTAS this corresponds'
)
print(
' to the station ID (e.g. wic)')
print(
' use "-o all" to get all stationids at a specific broker'
)
print(
'-i choose instrument(s) - only sensors containing'
)
print(
' the provided string are used: '
)
print(
' -i GSM will access GSM90_xxx and GSM19_xyz '
)
print(' Default is to use all')
print(
'-d set destination - std.out, db, file'
)
print(' default is std.out')
print('-v (no option)')
print(
' if provided: meta information will be added, revision'
)
print(
' automatically assigned.')
print(
' if not set: revision 0001 will be used, no meta.'
)
print(
'-l set location depending on destination'
)
print(' if d="file": provide path')
print(' if d="std.out": not used')
print(
'-c set mqtt communication credential keyword'
)
print(
'-q set mqtt quality of service (default = 0)'
)
print('-r set db credential keyword')
print(
'-s source protocol of data: default is mqtt'
)
print(' other options:')
print(
' -s wamp not yet implemented.'
)
print(
'-f offset values. Can either be "db" for'
)
print(
' applying delta values from db or a string'
)
print(
' of the following format (key:value):'
)
print(' -f "t1:3.234,x:45674.2"')
print(' other options:')
print('-m marcos configuration file ')
print(
' e.g. "/home/cobs/marcos.cfg"')
print('-n provide a integer number ')
print(
' "-d diff -i GSM": difference of two GSM will'
)
print(
' be calculated every n th step.'
)
print(
'-e provide a path to telegram configuration for '
)
print(
' sending critical log changes.')
print(
'-a additional MQTT translation library '
)
print('------------------------------------------------------')
print('Examples:')
print('1. Basic')
print(' python collector.py -b "192.168.0.100" -o wic')
print('2. Writing to file in directory "/my/path/"')
print(
' python collector.py -b "192.168.0.100" -d file -l "/my/path" -o wic'
)
print('3. Writing to file and stdout')
print(
' python collector.py -b "192.168.0.100" -d file,stdout -l "/tmp" -o wic'
)
print('4. Writing to db')
print(
' python collector.py -b "192.168.0.100" -d db -r mydb -o wic'
)
print(' python collector.py -d "db,file" -r testdb')
print('5. Using configuration file')
print(' python collector.py -m "/path/to/marcos.cfg"')
print('6. Overriding individual parameters from config file')
print(
' python collector.py -m "/path/to/marcos.cfg" -b "192.168.0.100"'
)
print(' (make sure that config is called first)')
print('7. Calculating differences/gradients on the fly:')
print(' python collector.py -d diff -i G823A -n 10')
print(
' (will calculate the diffs of two G823A every 10th record)')
sys.exit()
elif opt in ("-m", "--marcos"):
marcosfile = arg
print("Getting all parameters from configration file: {}".format(
marcosfile))
conf = acs.GetConf(marcosfile)
if not conf.get('logging', '') == '':
logging = conf.get('logging').strip()
if not conf.get('broker', '') == '':
broker = conf.get('broker').strip()
if not conf.get('mqttport', '') in ['', '-']:
port = int(conf.get('mqttport').strip())
if not conf.get('mqttdelay', '') in ['', '-']:
timeout = int(conf.get('mqttdelay').strip())
if not conf.get('mqttuser', '') in ['', '-']:
user = conf.get('mqttuser').strip()
if not conf.get('mqttqos', '') in ['', '-']:
try:
qos = int(conf.get('mqttqos').strip())
except:
qos = 0
if not conf.get('mqttcredentials', '') in ['', '-']:
credentials = conf.get('mqttcredentials').strip()
if not conf.get('blacklist', '') in ['', '-']:
blacklist = conf.get('blacklist').split(',')
blacklist = [el.strip() for el in blacklist]
if not conf.get('station', '') in ['', '-']:
stationid = conf.get('station').strip()
stid = stationid
if not conf.get('destination', '') in ['', '-']:
destination = conf.get('destination').strip()
if not conf.get('filepath', '') in ['', '-']:
location = conf.get('filepath').strip()
if not conf.get('databasecredentials', '') in ['', '-']:
dbcred = conf.get('databasecredentials').strip()
if not conf.get('revision', '') in ['', '-']:
destination = conf.get('revision').strip()
if not conf.get('offset', '') in ['', '-']:
offset = conf.get('offset').strip()
if not conf.get('debug', '') in ['', '-']:
debug = conf.get('debug').strip()
if debug in ['True', 'true']:
debug = True
else:
debug = False
if not conf.get('socketport', '') in ['', '-']:
try:
socketport = int(conf.get('socketport').strip())
except:
print(
'socketport could not be extracted from marcos config file'
)
socketport = 5000
if not conf.get('webport', '') in ['', '-']:
try:
webport = int(conf.get('webport').strip())
except:
print(
'webport could not be extracted from marcos config file'
)
webport = 8080
if not conf.get('webpath', '') in ['', '-']:
webpath = conf.get('webpath').strip()
if not conf.get('telegramconf', '') in ['', '-']:
telegramconf = conf.get('telegramconf').strip()
if not conf.get('addlib', '') in ['', '-']:
addlib = conf.get('addlib').strip().split(',')
source = 'mqtt'
elif opt in ("-b", "--broker"):
broker = arg
elif opt in ("-p", "--port"):
try:
port = int(arg)
except:
port = arg
elif opt in ("-t", "--timeout"):
try:
timeout = int(arg)
except:
timeout = arg
elif opt in ("-o", "--topic"):
stationid = arg
elif opt in ("-i", "--instrument"):
instrument = arg
elif opt in ("-s", "--source"):
source = arg
elif opt in ("-d", "--destination"):
destination = arg
elif opt in ("-l", "--location"):
location = arg
elif opt in ("-c", "--credentials"):
credentials = arg
elif opt in ("-v", "--revision"):
revision = "free"
elif opt in ("-r", "--dbcred"):
dbcred = arg
elif opt in ("-q", "--qos"):
try:
qos = int(arg)
except:
qos = 0
elif opt in ("-u", "--user"):
user = arg
elif opt in ("-P", "--password"):
password = arg
elif opt in ("-f", "--offset"):
offset = arg
elif opt in ("-n", "--number"):
number = arg
elif opt in ("-e", "--telegramconf"):
telegramconf = arg
elif opt in ("-a", "--addlib"):
addlib = arg.split(',')
elif opt in ("-U", "--debug"):
debug = True
if debug:
print("collector starting with the following parameters:")
print(
"Logs: {}; Broker: {}; Topic/StationID: {}; QOS: {}; MQTTport: {}; MQTTuser: {}; MQTTcredentials: {}; Data destination: {}; Filepath: {}; DB credentials: {}; Offsets: {}"
.format(logging, broker, stationid, qos, port, user, credentials,
destination, location, dbcred, offset))
try:
## Start Twisted logging system
## ----------------------------
if logging == 'sys.stdout':
log.startLogging(sys.stdout)
else:
try:
print(" -- Logging to {}".format(logging))
log.startLogging(open(logging, 'a'))
log.msg("----------------")
log.msg(" -> Logging to {}".format(logging))
except:
log.startLogging(sys.stdout)
log.msg("Could not open {}. Switching log to stdout.".format(
logging))
except:
print("Logging requires twisted module")
sys.exit()
try:
## Eventually import additional libraries
## ----------------------------
if addlib and len(addlib) > 0:
print("Importing additional library")
for lib in addlib:
exec("from libmqtt.{} import {}".format(lib, lib))
exec("c{} = {}()".format(lib, lib))
class_reference[lib] = eval("c{}".format(lib))
topic_identifiers[lib] = eval(
"c{}.topicidentifier".format(lib))
print("Imported library {}: Topic identifiers are {}".format(
lib, topic_identifiers[lib]))
except:
pass
if debug:
log.msg(
"Logs: {}; Broker: {}; Topic/StationID: {}; QOS: {}; MQTTport: {}; MQTTuser: {}; MQTTcredentials: {}; Data destination: {}; Filepath: {}; DB credentials: {}; Offsets: {}"
.format(logging, broker, stationid, qos, port, user, credentials,
destination, location, dbcred, offset))
log.msg("----------------")
log.msg(" Starting collector {}".format(__version__))
log.msg("----------------")
if not qos in [0, 1, 2]:
qos = 0
if 'stringio' in destination:
output = StringIO()
if 'file' in destination:
if location in [None, ''] and not os.path.exists(location):
log.msg(
'destination "file" requires a valid path provided as location'
)
log.msg(' ... aborting ...')
sys.exit()
if 'websocket' in destination:
if ws_available:
# 0.0.0.0 makes the websocket accessable from anywhere
global wsserver
wsserver = WebsocketServer(socketport, host='0.0.0.0')
wsThr = threading.Thread(target=wsThread, args=(wsserver, ))
# start websocket-server in a thread as daemon, so the entire Python program exits
wsThr.daemon = True
log.msg('starting WEBSOCKET on port ' + str(socketport))
wsThr.start()
# start webserver as process, also as daemon (kills process, when main program ends)
webPr = Process(target=webProcess, args=(webpath, webport))
webPr.daemon = True
webPr.start()
log.msg('starting WEBSERVER on port ' + str(webport))
else:
print(
"no webserver or no websocket-server available: remove 'websocket' from destination"
)
sys.exit()
if 'db' in destination:
if dbcred in [None, '']:
log.msg('destination "db" requires database credentials')
log.msg('to create them use method "addcred"')
log.msg('to provide use option -r like -r mydb')
sys.exit()
else:
try:
global db
if debug:
log.msg("Connecting database {} at host {} with user {}".
format(mpcred.lc(dbcred, 'db'),
mpcred.lc(dbcred, 'host'),
mpcred.lc(dbcred, 'user')))
db = mysql.connect(host=mpcred.lc(dbcred, 'host'),
user=mpcred.lc(dbcred, 'user'),
passwd=mpcred.lc(dbcred, 'passwd'),
db=mpcred.lc(dbcred, 'db'))
except:
log.msg(
'database {} at host {} with user {} could not be connected'
.format(mpcred.lc(dbcred, 'db'), mpcred.lc(dbcred, 'host'),
mpcred.lc(dbcred, 'user')))
log.msg(' ... aborting ...')
sys.exit()
if debug:
log.msg("Option u: debug mode switched on ...")
log.msg("------------------------------------")
log.msg("Destination: {} {}".format(destination, location))
if source == 'mqtt':
client = connectclient(broker, port, timeout, credentials, user,
password, qos)
client.loop_forever()
elif source == 'wamp':
log.msg("Not yet supported! -> check autobahn import, crossbario")
else:
log.msg("Additional protocols can be added in future...")
clientip = '138.22.188.181'
# Path of MARTAS directory on martas machine
martaspath = '/home/cobs/MARTAS'
# Path of MARCOS directory
homedir = '/home/cobs'
defaultuser = '******'
# Provide Station code
stationid = 'WIC'
# Select destination (file or db) - Files are saved in .../MARCOS/MartasFiles/
dest = 'db'
# For Testing purposes - Print received data to screen:
printdata = False
# Please make sure that the db and scp connection data is stored
# within the credential file -otherwise provide this data directly
dbhost = mpcred.lc('cobsdb','host',path='/home/cobs/.magpycred')
dbuser = mpcred.lc('cobsdb','user',path='/home/cobs/.magpycred')
dbpasswd = mpcred.lc('cobsdb','passwd',path='/home/cobs/.magpycred')
dbname = mpcred.lc('cobsdb','db',path='/home/cobs/.magpycred')
scpuser = mpcred.lc('ceres','user',path='/home/cobs/.magpycred')
scppasswd = mpcred.lc('ceres','passwd',path='/home/cobs/.magpycred')
# You can add to the credential file by using:
# mpcred.cc('transfer','myshortcut',user='******',passwd='mypasswd',address='no-specific')
# and than read it by scpuser = mpcred.lc('myshortcut','myuser')
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# do necessary changes above
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
print dbhost, dbuser, dbpasswd, dbname, scpuser
logfile = os.path.join(homedir,'MARCOS','Logs','marcos.log')
def automated_storm_plot(magdata, satdata_1m, satdata_5m, mag_results, sat_results,
plotvars=['x','var2', 'var1'], logpflux=True, estimate=False, savepath='',upload=False):
date = datetime.strftime(mag_results['ssctime'],'%Y-%m-%d')
ssctime = mag_results['ssctime']
ssctime_st = datetime.strftime(ssctime,'%H:%M:%S')
ssctime_st_long = datetime.strftime(ssctime,'%Y-%m-%dT%H:%M:%S')
acetime = sat_results['satssctime']
acetime_st = datetime.strftime(acetime,'%H:%M:%S')
acetime_st_long = datetime.strftime(acetime,'%Y-%m-%dT%H:%M:%S')
if logpflux:
pflux = satdata_5m.ndarray[KEYLIST.index('var1')].astype(float)
logpflux = np.log10(pflux)
satdata_5m._put_column(np.asarray(logpflux), 'var1')
satdata_5m.header['col-var1'] = 'log10(P-flux 47-68 keV)\n '
satdata_5m.header['unit-col-var1'] = 'p/cm2-s-ster-MeV'
magdata = magdata.nfilter()
dH = mag_results['amp']
h_max = magdata._get_max(plotvars[0])
h_min = magdata._get_min(plotvars[0])
v_wind = sat_results['vwind']
v_max = satdata_1m._get_max(plotvars[1])
v_min = satdata_1m._get_min(plotvars[1])
startdate = datetime.strftime(num2date(magdata.ndarray[0][0]), '%Y-%m-%d')
enddate = datetime.strftime(num2date(magdata.ndarray[0][-1]), '%Y-%m-%d')
if startdate == enddate:
datestr = startdate
else:
datestr = startdate+' - '+enddate
fig = plotStreams([magdata, satdata_1m, satdata_5m], [[plotvars[0]], [plotvars[1]], [plotvars[2]]],
noshow=True,
plottitle="Magnetic (Conrad Obs) and satellite (ACE SWEPAM+EPAM) data\n(%s)" % datestr)
axes = gcf().get_axes()
# Draw lines at detection points:
axes[0].axvline(x=date2num(ssctime),color='red',ls='--',lw=2,zorder=0, clip_on=False)
axes[1].axvline(x=date2num(acetime),color='gray',ls='--',lw=2,zorder=0, clip_on=False)
axes[2].axvline(x=date2num(acetime),color='gray',ls='--',lw=2,zorder=0, clip_on=False)
# Give larger borders to lines:
axes[0].axvline(x=date2num(ssctime), ymin=0, ymax=0.94, color='red',ls='-',lw=15,zorder=0,alpha=0.3, clip_on=False)
axes[1].axvline(x=date2num(acetime), ymin=0, ymax=0.94, color='gray',ls='-',lw=15,zorder=0,alpha=0.3, clip_on=False)
axes[2].axvline(x=date2num(acetime), ymin=0.07, ymax=0.94, color='gray',ls='-',lw=15,zorder=0,alpha=0.3, clip_on=False)
# Annotate with automatically detected variables:
if estimate:
axes[0].text(0.50, 0.90, "No SSC detected!",
verticalalignment='top', horizontalalignment='left',
transform=axes[0].transAxes, color='red', style='italic')
magstring = "Expected SSC:\n%s UTC\ndH = ??? nT" % (ssctime_st)
else:
magstring = "SSC:\n%s UTC\ndH = %.1f nT" % (ssctime_st, dH)
axes[0].text(0.78, 0.90, magstring,
verticalalignment='top', horizontalalignment='left',
transform=axes[0].transAxes,
bbox=dict(boxstyle="square",fc='1.0')
)
axes[1].text(0.78, 0.90, "CME at ACE:\n%s UTC\nv = %.0f km/s" % (acetime_st, v_wind),
verticalalignment='top', horizontalalignment='left',
transform=axes[1].transAxes,
bbox=dict(boxstyle="square",fc='1.0')
)
# Format time to fit nicely on x-axis:
myFmt = DateFormatter('%H:%M')
axes[2].xaxis.set_major_formatter(myFmt)
# Save to output file:
#plt.show()
magoutfile = os.path.join(savepath, "stormplot_%s.png" % date)
plt.savefig(magoutfile)
if upload:
cred = 'cobshomepage'
address=mpcred.lc(cred,'address')
user=mpcred.lc(cred,'user')
passwd=mpcred.lc(cred,'passwd')
port=mpcred.lc(cred,'port')
remotepath = 'zamg/images/graphs/spaceweather/storms'
scptransfer(magoutfile,'94.136.40.103:'+remotepath,passwd)
command = ["cp", magoutfile, "/srv/products/graphs/spaceweather/storms/"]
print "Executing command %s" % command
call(command)
upload = True
def ConnectDatabases(config={}, debug=False):
"""
DESCRIPTION:
Database connection
"""
connectdict = {}
config['primaryDB'] = None
dbcreds = config.get('dbcredentials')
if not isinstance(dbcreds, list):
dbcreds = [dbcreds]
# First in list is primary
for credel in dbcreds:
# Get credentials
dbpwd = mpcred.lc(credel, 'passwd')
dbhost = mpcred.lc(credel, 'host')
dbuser = mpcred.lc(credel, 'user')
dbname = mpcred.lc(credel, 'db')
# Connect DB
if dbhost:
try:
if debug:
print(
" -- Connecting to database {} ...".format(credel))
print(" -- {} {} {}".format(dbhost, dbuser, dbname))
connectdict[credel] = mysql.connect(host=dbhost,
user=dbuser,
passwd=dbpwd,
db=dbname)
if debug:
print("...success")
except:
pass
if len(connectdict) == 0:
print(" No database found - aborting")
sys.exit()
else:
if debug:
print(" -- at least on db could be connected")
if connectdict.get(dbcreds[0], None):
if debug:
print(" -- primary db is available: {}".format(dbcreds[0]))
config['primaryDB'] = connectdict[dbcreds[0]]
else:
print(
" Primary database not available - selecting alternative as primary"
)
for el in dbcreds:
if connectdict.get(el, None):
config['primaryDB'] = connectdict[el]
print(" -> selected database {} as primary".format(el))
break
config['conncetedDB'] = connectdict
return config
def main(argv):
statusmsg = {}
jobs = ''
tele = ''
sendlogpath = ''
try:
opts, args = getopt.getopt(argv, "hj:m:t:", [
"jobs=",
"memory=",
"telegram=",
])
except getopt.GetoptError:
print('file_upload.py -j <jobs> -m <memory> -t <telegram>')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('-------------------------------------')
print('Description:')
print(
'-- file_uploads.py send data to any destination of your choice --'
)
print(
'-----------------------------------------------------------------'
)
print(
'file_uploads is a python wrapper allowing to send either by')
print('ftp, scp, sftp using a similar inputs.')
print('It can hanlde several different uploads at once.')
print('Upload parameters have to be provided in a json structure.')
print('file_uploads requires magpy >= 0.9.5.')
print('-------------------------------------')
print('Usage:')
print('file_uploads.py -j <jobs> -m <memory> -t <telegram>')
print('-------------------------------------')
print('Options:')
print('-j (required) : a json structure defining the uplaods')
print('-m (required) : a path for "memory"')
print('-------------------------------------')
print('Example of jobs structure:')
print(
'{"wicadjmin":{"path":"/home/leon/Tmp/Temp","destinations":{"gleave":{"type":"sftp", "path" : "/uploads/all-obs"}},'
)
print(
'"log":"/home/leon/Tmp/wicadjart.log","endtime":"utcnow","starttime":2}}'
)
print('! please note endtime is usually utcnow')
print(
'! starttime is given as a timedelta in days toward endtime, an integer is required'
)
print('-------------------------------------')
print('Application:')
print(
'python3 file_uploads.py -j /my/path/uploads.json -m /tmp/sendmemory.json'
)
sys.exit()
elif opt in ("-j", "--jobs"):
jobs = arg
elif opt in ("-m", "--memory"):
sendlogpath = arg
elif opt in ("-t", "--telegram"):
tele = arg
if tele:
# ################################################
# Telegram Logging
# ################################################
## New Logging features
coredir = os.path.abspath(os.path.join(scriptpath, '..', 'core'))
sys.path.insert(0, coredir)
from martas import martaslog as ml
# tele needs to provide logpath, and config path ('/home/cobs/SCRIPTS/telegram_notify.conf')
logpath = '/var/log/magpy/mm-fu-uploads.log'
if jobs == '':
print('Specify a valid path to a jobs dictionary (json):')
print('-- check file_uploads.py -h for more options and requirements')
sys.exit()
else:
if os.path.isfile(jobs):
with open(jobs, 'r') as file:
workdictionary = json.load(file)
else:
print('Specify a valid path to a jobs dictionary (json):')
print(
'-- check file_uploads.py -h for more options and requirements'
)
sys.exit()
# make fileupload an independent method importing workingdictionary (add to MARTAS?)
if not sendlogpath:
sendlogpath = '/tmp/lastupload.json'
"""
Main Prog
"""
try:
for key in workdictionary:
name = "FileUploads-{}".format(key)
print("DEALING with ", key)
lastfiles = {}
fulldict = {}
if os.path.isfile(sendlogpath):
with open(sendlogpath, 'r') as file:
fulldict = json.load(file)
lastfiles = fulldict.get(key)
# lastfiles looks like: {'/path/to/my/file81698.txt' : '2019-01-01T12:33:12', ...}
if not lastfiles == {}:
print("opened memory")
pass
sourcepath = workdictionary.get(key).get('path')
extensions = workdictionary.get(key).get('extensions', [])
namefractions = workdictionary.get(key).get('namefractions', [])
# Test if sourcepath is file
starttime = workdictionary.get(key).get('starttime',
datetime(1777, 4, 30))
endtime = workdictionary.get(key).get('endtime', datetime.utcnow())
if endtime in ["utc", "now", "utcnow"]:
endtime = datetime.utcnow()
if isinstance(starttime, int):
starttime = datetime.utcnow() - timedelta(days=starttime)
newfiledict, alldic = getchangedfiles(sourcepath, lastfiles,
starttime, endtime,
extensions, namefractions)
print("Found new: {} and all {}".format(newfiledict, alldic))
for dest in workdictionary.get(key).get('destinations'):
print(" -> Destination: {}".format(dest))
address = mpcred.lc(dest, 'address')
user = mpcred.lc(dest, 'user')
passwd = mpcred.lc(dest, 'passwd')
port = mpcred.lc(dest, 'port')
destdict = workdictionary.get(key).get('destinations')[dest]
proxy = destdict.get('proxy', None)
#print (destdict)
if address and user and newfiledict:
for nfile in newfiledict:
print(" -> Uploading {} to dest {}".format(
nfile, dest))
success = uploaddata(nfile,
destdict.get('path'),
destdict.get('type'),
address,
user,
passwd,
port,
proxy=proxy,
logfile=destdict.get(
'logfile', 'stdout'))
print(" -> Success", success)
if not success:
#remove nfile from alldic
# thus it will be retried again next time
print(
" !---> upload of {} not successful: keeping it in todo list"
.format(nfile))
del alldic[nfile]
fulldict[key] = alldic
writecurrentdata(sendlogpath, fulldict)
statusmsg[name] = "uploading data succesful"
except:
statusmsg[name] = "error when uploading files - please check"
if tele:
print(statusmsg)
martaslog = ml(logfile=logpath, receiver='telegram')
martaslog.telegram[
'config'] = '/home/cobs/SCRIPTS/telegram_notify.conf'
martaslog.msg(statusmsg)
print("----------------------------------------------------------------")
print("file upload app finished")
print("----------------------------------------------------------------")
print("SUCCESS")
#!/bin/env/python
from magpy.stream import read
import magpy.mpplot as mp
from magpy.database import *
import magpy.opt.cred as mpcred
dbpasswd = mpcred.lc('cobsdb', 'passwd')
db = mysql.connect(host="138.22.188.195",
user="******",
passwd=dbpasswd,
db="cobsdb")
vario1 = read('/srv/archive/WIC/LEMI036_1_0002/raw/*2016-02-28.bin')
vario2 = read('/srv/archive/WIC/LEMI025_22_0002/raw/*2016-02-28.bin')
print vario1.length(), vario2.length()
mp.plotStreams([vario1, vario2], [['z'], ['z']])
cmetime = cmetime.astimezone(utczone)
sscdata = reportdata[skip+2].split()
ssctime = datetime.strptime(sscdata[5]+'T'+sscdata[6], '%Y-%m-%dT%H:%M:%S')
ssctime = ssctime.replace(tzinfo=cetzone)
ssctime = ssctime.astimezone(utczone)
sscamp = np.nan
vwind = float(reportdata[skip+3].split()[4])
estimate = True
if (now - ssctime).seconds > 24.*60.*60.:
print("Time since SSC exceeds 24 hours. Exiting.")
sys.exit()
basepath = '/srv/archive'
dbcred = 'cobsdb'
dbhost = mpcred.lc(dbcred, 'host')
dbuser = mpcred.lc(dbcred, 'user')
dbpasswd = mpcred.lc(dbcred, 'passwd')
dbname = mpcred.lc(dbcred, 'db')
try:
inst = 'FGE_S0252_0001_0001'
db = mysql.connect(host=dbhost,user=dbuser,passwd=dbpasswd,db=dbname)
magdata = readDB(db, inst, starttime=cmetime-timedelta(hours=1), endtime=endtime)
if len(magdata.ndarray[0]) == 0:
raise ValueError("No data available from database.")
except:
print("Reading data from archive...")
magdata = read(os.path.join(basepath,"WIC","FGE_S0252_0001","FGE_S0252_0001_0001","FGE_S0252_0001_0001_*"),
starttime=cmetime-timedelta(hours=1), endtime=now)
satdata_1m = read(os.path.join(basepath,"external","esa-nasa","ace","collected","ace_1m_*"),
