def CreateDiagram(datastream,config={},statusmsg={}, endtime=datetime.utcnow(), debug=False):
"""
DESCRIPTION
Create diagram of adjusted data
"""
figpath = config.get("magfigurepath")
daystodeal = config.get("daystodeal")
date = datetime.strftime(endtime,"%Y-%m-%d")
yesterd2 = datetime.strftime(endtime-timedelta(days=daystodeal),"%Y-%m-%d")
success = True
name5 = "{}-CreateDiagramVario".format(config.get('logname','Dummy'))
# Create realtime diagram and upload to webpage WDC
try:
print (" Creating plot")
pnd = datastream._select_timerange(starttime=datetime.strftime(endtime-timedelta(days=daystodeal),"%Y-%m-%d"))
pst = DataStream([LineStruct()],datastream.header,pnd)
pst = pst.xyz2hdz()
mp.plotStreams([pst],[['x','y','z','f']], gridcolor='#316931',fill=['x','z','f'],confinex=True, fullday=True, opacity=0.7, plottitle='Geomagnetic variation (until %s)' % (datetime.utcnow().date()),noshow=True)
print (" -- Saving diagram to products folder")
pltsavepath = os.path.join(figpath,"magvar_{}.png".format(date))
plt.savefig(pltsavepath)
statusmsg[name5] = 'creating and saving graph successful'
print (" -> Done")
except:
statusmsg[name5] = 'failed to save data - remount necessary?'
success = False
return success, statusmsg
def CreateDiagram(streamlist,keylist, filllist=None, colorlist=None, paddinglist=None, annotatelist=None, symbollist=[], specialdict=None, gridcolor='#316931', confinex=True, fullday=True, opacity=0.7, bartrange=0.06, style='magpy',show=False,fullplotpath='',debug=False):
if debug:
show=True
if style in ['magpy','MagPy','MAGPY']:
# TODO Union colorlist, etc
mp.plotStreams(streamlist,keylist, fill=filllist, colorlist=colorlist, padding=paddinglist, annotate=annotatelist, symbollist=symbollist, specialdict=specialdict, gridcolor=gridcolor, confinex=confinex, fullday=fullday, opacity=opacity, noshow=True)
else:
print (" -> unkown plot style ... doing nothing")
return False
print (" ... plotting done")
if fullplotpath:
print (" ... saving plot to {}".format(fullplotpath))
if not debug:
plt.savefig(fullplotpath)
else:
print (" ... debug selected : otherwise would save figure to {}".format(fullplotpath))
if show:
print (" ... debug mode or show selected - showing plot")
plt.show()
print (" -> finished")
return True
def gravity_graph(db,config={},starttime=datetime.utcnow()-timedelta(days=4),endtime=datetime.utcnow(), debug=False)):
data = read(os.path.join(path,'F1*'),starttime=starttime, endtime=endtime)
# get last values of filt
lastval = data.ndarray[1][-1]
print (lastval)
if lastval < 0:
img = imread(os.path.join(path2images,"ebbe_neu.jpg"))
else:
img = imread(os.path.join(path2images,"flut_neu.jpg"))
fig = mp.plotStreams([data],[['x','z']], fill=['x'], colorlist=['g','w'], gridcolor='#316931', opacity=0.3,singlesubplot=True,noshow=True)
fig.set_size_inches(9.56, 2.19, forward=True)
#fig.set_size_inches(17.56, 2.19, forward=True)
fig.patch.set_facecolor('black')
fig.subplots_adjust(left=0.01, right=0.99, top=0.99, bottom=0.01)
# remove boundary and rotate image
ax = fig.axes
for a in ax:
a.axis('off')
newax = fig.add_axes([0.0, 0.0, 1.0, 1.0],anchor='SW',zorder=-1)
newax.imshow(img,origin='upper')
newax.axis('off')
savepath2 = savepath = "/srv/products/graphs/title/title_gravity.png"
plt.savefig(savepath2)
return True
#!/bin/env/python
from magpy.stream import read
import magpy.mpplot as mp
from magpy.database import *
import magpy.opt.cred as mpcred
dbpasswd = mpcred.lc('cobsdb', 'passwd')
db = mysql.connect(host="138.22.188.195",
user="******",
passwd=dbpasswd,
db="cobsdb")
vario1 = read('/srv/archive/WIC/LEMI036_1_0002/raw/*2016-02-28.bin')
vario2 = read('/srv/archive/WIC/LEMI025_22_0002/raw/*2016-02-28.bin')
print vario1.length(), vario2.length()
mp.plotStreams([vario1, vario2], [['z'], ['z']])
def CreateDiagram(config={},
endtime=datetime.utcnow(),
dayrange=1,
debug=False):
"""
Skeleton
"""
# basic parameter
starttime = endtime - timedelta(days=dayrange)
savepath = "/srv/products/graphs/tilt/tilt_%s.png" % date
# READ data
try:
dscovr_plasma = read(
"http://services.swpc.noaa.gov/products/solar-wind/plasma-3-day.json"
)
dscovr_mag = read(
"http://services.swpc.noaa.gov/products/solar-wind/mag-3-day.json")
except Exception as e:
print("Reading data failed ({})!".format(e))
return False
try:
kp = read(path_or_url=os.path.join('/srv/archive/external/gfz', 'kp',
'gfzkp*'))
kp = kp.trim(starttime=starttime, endtime=endtime)
except Exception as e:
print("Reading data failed ({})!".format(e))
return False
# MODIFY data
startstr = num2date(dscovr_mag.ndarray[0][0])
endstr = num2date(dscovr_mag.ndarray[0][-1])
print("Plotting from %s to %s" % (startstr, endstr))
dscovr_plasma.header['col-var1'] = 'Density'
dscovr_plasma.header['unit-col-var1'] = 'p/cc'
dscovr_plasma.header['col-var2'] = 'Speed'
dscovr_plasma.header['unit-col-var2'] = 'km/s'
dscovr_mag.header['col-z'] = 'Bz'
dscovr_mag.header['unit-col-z'] = 'nT'
# PLOT data
mp.plotStreams(
[kp, dscovr_mag, dscovr_plasma], [['var1'], ['z'], ['var1', 'var2']],
confinex=True,
bartrange=0.06,
symbollist=['z', '-', '-', '-'],
specialdict=[{
'var1': [0, 9]
}, {}, {}],
plottitle="Solar and global magnetic activity (GFZ + DSCOVR data)",
outfile=os.path.join(solarpath),
noshow=True)
if not debug:
print(" ... saving now")
plt.savefig(savepath)
print("Plot successfully saved to {}.".format(savepath))
else:
print(" ... debug mode - showing plot")
plt.show()
return True
print("Running ionit analysis ...")
rdata = read(os.path.join(radonpath, rfile), starttime='2016-12-01')
#print (rdata.ndarray[0])
#mp.plot(rdata)
mdata = read('/srv/products/data/meteo/meteo*', starttime='2016-12-01')
#mp.plot(mdata)
#sys.exit()
# Run the analysis and plots
imdata = read(os.path.join(impath, '*'))
print(imdata.length())
#mp.plot(imdata)
st, et = imdata._find_t_limits()
st = startofborehole
imdata = imdata.trim(starttime=st, endtime=et)
tdata = read(os.path.join(envpath, tfile), starttime=st, endtime=et)
pdata = read(os.path.join(envpath, pfile), starttime=st, endtime=et)
rdata = read(os.path.join(radonpath, rfile), starttime=st, endtime=et)
mdata = read(os.path.join(meteopath, mfile), starttime=st, endtime=et)
#mp.plot(envdata)
mp.plotStreams([imdata, tdata, pdata, rdata, mdata],
[['y', 't1'], ['t1', 'var1'], ['x'], ['x'], ['f']],
noshow=True)
pltsavepath = "/srv/projects/ionit/graphs/ionit-timeseries.png"
plt.savefig(pltsavepath)
print ("Flagging with earthquake data failed")
pass
# providing some Info on content
#print data._get_key_headers()
#matplotlib.rc('font', **font)
print grav.length()
grav.header['col-x'] = ''
grav.header['col-z'] = ''
res.header['col-z'] = ''
grav.header['unit-col-x'] = ''
grav.header['unit-col-z'] = ''
res.header['unit-col-z'] = ''
if grav.length()[0] > 1 and res.length()[0]> 1:
mp.plotStreams([grav,res],[['x','z'],['z']], gridcolor='#316931',fill=['x'], annotate=[[True, False],[False]],confinex=True, fullday=True, opacity=0.7, plottitle='Gravity variation',noshow=True, labels=[['G [nm/s2]','Pressure [hPa]'],['Residual [nm/s2]']])
elif grav.length()[0] > 1:
mp.plotStreams([grav],[['x','z']], gridcolor='#316931',confinex=True, fullday=True,plottitle='Gravity variation' ,noshow=True, labels=[['G [nm/s2]','Pressure [hPa]']])
else:
print ("No data available")
plt.rc('font', **font)
#upload
savepath = "/srv/products/graphs/gravity/gravity_%s.png" % date
plt.savefig(savepath)
cred = 'cobshomepage'
address=mpcred.lc(cred,'address')
user=mpcred.lc(cred,'user')
passwd=mpcred.lc(cred,'passwd')
sub = subtractStreams(streamlist[subset[0]], streamlist[subset[1]])
ms = sub.mean('f', meanfunction='std')
print "Delta F:", sub.mean('f', std=True)
if np.isnan(ms):
ms = 9999
subset = [el for el in subset]
subset.append(ms)
meanlst.append(subset)
# Create plot to be uploaded later
title = streamlist[subset[0]].header.get(
'SensorID', 'None') + '-' + streamlist[subset[1]].header.get(
'SensorID', 'None')
try:
mp.plotStreams(
[streamlist[subset[0]], streamlist[subset[1]], sub],
[['f'], ['f'], ['f']],
plottitle=title,
noshow=True)
savepath = "/srv/products/graphs/magnetism/deltaf%d%d_%s.png" % (
subset[0], subset[1], date)
plt.savefig(savepath)
except:
pass
print("Stddev list:", meanlst)
std = [el[2] for el in meanlst]
mi = np.min(std)
subs = meanlst[std.index(mi)]
streamlist = [el for idx, el in enumerate(streamlist) if idx in subs]
optimalf = np.min([int(subs[0]), int(subs[1])])
print("Optimal F instrument is {}".format(scalalist[optimalf])
def mag_graph(db,
config={},
starttime=datetime.utcnow() - timedelta(days=3),
endtime=datetime.utcnow(),
debug=False):
msg = 'mag graph successful'
variosens = config.get('primaryVario')
scalarsens = config.get('primaryScalar')
varioinst = config.get('primaryVarioInst')
scalarinst = config.get('primaryScalarInst')
if debug:
print('Getting Solare image')
print('TODO Move that job to aldebaran')
sohopath = '/srv/archive/external/esa-nasa/soho/'
sohoname = 'EIT304_latest.jpg'
sohonewname = 'EIT304_{}.jpg'.format(
datetime.strftime(datetime.utcnow(), "%Y-%m-%d_%H"))
if debug:
print("New name:", sohonewname)
try:
urllib.request.urlretrieve(
"https://sohowww.nascom.nasa.gov/data/realtime/eit_304/512/latest.jpg",
sohoname)
except:
msg = "Latest soho image not available"
try:
if not os.path.exists(sohopath):
os.makedirs(sohopath)
shutil.copyfile(sohoname, os.path.join(sohopath, sohonewname))
except:
print("copying failed")
print('Reading data from primary instrument')
try:
data = readDB(db, varioinst, starttime=starttime)
kvals = data.k_fmi(k9_level=500)
except:
msg = 'problem reading data'
img = imread("/home/cobs/ANALYSIS/TitleGraphs/EIT304_latest.jpg")
print('Plotting streams')
fig = mp.plotStreams([kvals, data], [['var1'], ['x']],
specialdict=[{
'var1': [0, 18]
}, {}],
symbollist=['z', '-'],
bartrange=0.06,
colorlist=['k', 'y'],
gridcolor='#316931',
singlesubplot=True,
noshow=True,
opacity=0.5)
fig.set_size_inches(9.56, 2.19, forward=True)
fig.patch.set_facecolor('black')
fig.subplots_adjust(left=0.01, right=0.99, top=0.99, bottom=0.01)
ax = fig.axes
for a in ax:
a.axis('off')
#a.set_frame_on(False)
print(" p1")
maxk = kvals._get_max('var1')
if maxk >= 6:
img2 = imread("/home/cobs/ANALYSIS/TitleGraphs/polarlichter.v02.jpg")
newax2 = fig.add_axes([0.0, 0.0, 1.0, 1.0], anchor='SE', zorder=-1)
newax2.imshow(img2, origin='upper')
newax2.axis('off')
else:
img2 = imread("/home/cobs/ANALYSIS/TitleGraphs/hyades.v03.jpg")
newax2 = fig.add_axes([0.0, 0.0, 1.0, 1.0], anchor='SE', zorder=-1)
newax2.imshow(img2, origin='upper')
newax2.axis('off')
#if debug:
#print (" p2")
newax = fig.add_axes([0.0, 0.0, 1.0, 1.0], anchor='SW', zorder=-1)
newax.imshow(img, origin='upper')
newax.axis('off')
#if debug:
#plt.show()
savepath2 = "/srv/products/graphs/title/title_mag.png"
plt.savefig(savepath2)
print("Save to {} done".format(savepath2))
def weather_graph(db,
config={},
starttime=datetime.utcnow() - timedelta(days=4),
endtime=datetime.utcnow(),
debug=False):
data = readDB(db, 'METEO_T7_0001_0001', starttime=starttime)
month = endtime.month
if month in [12, 1, 2, 3]:
img = imread(os.path.join(scriptpath, "winter.jpg"))
elif month in [4, 5, 6]:
img = imread(os.path.join(scriptpath, "spring.jpg"))
elif month in [7, 8, 9]:
img = imread(os.path.join(scriptpath, "summer.jpg"))
else:
img = imread(os.path.join(scriptpath, "autumn.jpg"))
clean = True
if clean:
res = data.steadyrise('dx',
timedelta(minutes=60),
sensitivitylevel=0.002)
data = data._put_column(res,
't2',
columnname='Niederschlag',
columnunit='mm/1h')
data = data.flag_outlier(keys=['f', 't1', 'var5', 'z'],
threshold=4.0,
timerange=timedelta(hours=2))
data = data.remove_flagged()
snow = data._get_column('z')
snowmax = np.max([el for el in snow if not isnan(el)])
if snowmax < 200:
snowmax = 200
rain = data._get_column('t2')
rainmax = np.max([el for el in rain if not isnan(el)])
if rainmax < 15:
rainmax = 15
#month = 1
if month in [11, 12, 1, 2, 3]:
fig = mp.plotStreams([data], [['z', 'f']],
fill=['z'],
specialdict=[{
'z': [0, snowmax]
}, {
'f': [-10, 40]
}],
colorlist=['w', 'r'],
gridcolor='#316931',
opacity=0.5,
singlesubplot=True,
noshow=True)
else:
fig = mp.plotStreams([data], [['t2', 'f']],
fill=['t2'],
specialdict=[{
't2': [0, rainmax]
}, {
'f': [-30, 20]
}],
colorlist=['b', 'r'],
gridcolor='#316931',
opacity=0.5,
singlesubplot=True,
noshow=True)
fig.set_size_inches(9.56, 2.19, forward=True)
fig.patch.set_facecolor('black')
fig.subplots_adjust(left=0.01, right=0.99, top=0.99, bottom=0.01)
# remove boundary and rotate image
ax = fig.axes
for a in ax:
a.axis('off')
newax = fig.add_axes([0.0, 0.0, 1.0, 1.0], anchor='SW', zorder=-1)
newax.imshow(img, origin='upper')
newax.axis('off')
savepath2 = savepath = "/srv/products/graphs/title/title_weather.png"
plt.savefig(savepath2)
return True
mydir = "/srv/projects/geoelectrics/Daten-Geoelektrik/"
data = DataStream()
for fi in os.listdir(mydir):
if fi.startswith("geoelektrik_mon"):
print(os.path.join(mydir, fi))
filename = os.path.join(mydir, fi)
cdata = readLMEL(filename)
data.extend(cdata.container, cdata.header, cdata.ndarray)
data = data.sorting()
data = data.get_gaps()
data = data.offset({'time': timedelta(hours=-2)})
"""
data = DataStream()
data1 = readLMEL('/home/leon/CronScripts/Geoelektrik/geoelektrik_monitoring-2017-04-01.txt')
data.extend(data1.container,data1.header,data1.ndarray)
data2 = readLMEL('/home/leon/CronScripts/Geoelektrik/geoelektrik_monitoring-2017-05-04.txt')
data.extend(data2.container,data2.header,data2.ndarray)
"""
#mp.plot(data)
meteo = read('/srv/products/data/meteo/meteo-1min*')
print(meteo._get_key_headers())
#mp.plot(meteo)
# Create a small report and add the following plot -> send out per mail ???
mp.plotStreams([data, meteo], variables=[['x'], ['y', 'z']], noshow=True)
pltsavepath = "/srv/projects/geoelectrics/graphs/wenner_timeseries.png"
plt.savefig(pltsavepath)
#print ("Result", data)
except:
roomtest = False
# Read and filter CO2
try:
co = read(mq135path, starttime=start, endtime=end)
co = co.filter()
if co.length()[0] > 0:
cotest = True
else:
cotest = False
except:
cotest = False
if step2:
if cotest and roomtest and meteotest and gammatest and ionotest:
mp.plotStreams([iono, gamma, meteo, room, co],
[['x'], ['t1'], ['f'], ['t1', 'var2'], ['var5']])
statusdict['project_iono_graph'] = 'full data plot'
elif cotest and roomtest and gammatest and ionotest:
mp.plotStreams([iono, gamma, room, co],
[['x'], ['t1'], ['t1', 'var2'], ['var5']])
statusdict['project_iono_graph'] = 'reduced plot - check meteo'
elif cotest and roomtest and gammatest and meteotest:
mp.plotStreams([gamma, meteo, room, co],
[['t1'], ['f'], ['t1', 'var2'], ['var5']])
statusdict['project_iono_graph'] = 'reduced plot - check iono'
elif cotest and roomtest and gammatest:
mp.plotStreams([gamma, room, co], [['t1'], ['t1', 'var2'], ['var5']])
statusdict[
'project_iono_graph'] = 'minimal plot - check iono and meteo'
else:
statusdict['project_iono_graph'] = 'not enough data'