def siata_remote_data_to_transfer(start, end, *args, **kwargs):
remote = cpr.Nivel(**cpr.info.REMOTE)
codigos_str = '(' + str(list(local.infost.index)).strip('[]') + ')'
parameters = tuple([codigos_str, local.fecha_hora_query(start, end)])
df = remote.read_sql('SELECT * FROM datos WHERE cliente in %s and %s' %
parameters)
return df
def plot_level(serie,codigo,resolution,folder,path = '/media/nicolas/Home/Jupyter/MarioLoco'):
try: # will be argument later on
obj = cpr.Nivel(codigo = codigo,user='******',passwd='s@mple_p@ss',SimuBasin=False)
levantamiento = pd.read_csv('%s/ultimos_levantamientos/%s.csv'%(path,codigo),index_col=0)
filepath = '%s/%s.png'%(path+'/real_time/'+folder,obj.info.slug)
obj.plot_operacional(serie,levantamiento,resolution,filepath=filepath)
r = os.system('scp %s [email protected]:/var/www/mario/realTime/%s'%(filepath,folder))
except:
print 'error in plot %s'%codigo
def cron():
def convert_to_risk(df):
df = self.risk_df(df)
return df[df.columns.dropna()]
#PROCESS
self = cpr.Nivel(codigo=99,
user='******',
passwd='s@mple_p@ss',
SimuBasin=False)
end = datetime.datetime.now()
start = end - datetime.timedelta(minutes=150)
df = self.level_all(start, end, calidad=True)
risk_df = convert_to_risk(df.copy())
def select(pos):
try:
colors = ['green', '#F7DD1E', '#F9A43E', '#FC3A3A', '#FC3A3A']
return colors[pos]
except IndexError:
return '#D8E0E8'
filepath = "http://siata.gov.co/mario/realTime/tres_horas/"
data_list = []
for codigo in risk_df.index[::-1]:
positions = np.array(risk_df.loc[codigo].values, int)
color = map(lambda x: select(x), positions)
data = pd.DataFrame(color, index=risk_df.loc[codigo].index)
data['hour'] = risk_df.loc[codigo].index.strftime('%H:%M')
data['nombre'] = "%s | " % codigo + str(self.infost.loc[codigo,
'nombre'])
data['path'] = filepath + self.infost.loc[codigo, 'slug'] + '.png'
data['location'] = self.infost.loc[codigo, 'municipio']
data['telefono'] = 'No'
data['celular'] = 'No'
data['waterlevel'] = df.T.loc[codigo]
data.columns = [
'color', 'hour', 'name', 'path', 'location', 'phone', 'mobile',
'waterlevel'
]
data_list.append(data)
data_list = pd.concat(data_list)
data_list.name = data_list.name.str.replace(' - Nivel', '')
filename = 'heatmap_data.csv'
data_list.to_csv(filename)
statement = 'scp %s [email protected]:/var/www/mario/realTime/risk_levels_chart/%s' % (
filename, filename)
print(os.system(statement))
print(os.system(statement))
print(os.system(statement))
def convert_series_to_risk(level):
'''level: pandas Series, index = codigos de estaciones'''
risk = level.copy()
colors = ['green', 'gold', 'orange', 'red', 'red', 'black']
for codigo in level.index:
try:
risks = cpr.Nivel(codigo=codigo,
user='******',
passwd='s@mple_p@ss').risk_levels
risk[codigo] = colors[int(
self.convert_level_to_risk(level[codigo], risks))]
except:
risk[codigo] = 'black'
return risk
def process_multiple_plots_looping():
self = cpr.Nivel(codigo=99,
user='******',
passwd='s@mple_p@ss',
SimuBasin=False)
for codigo in df.columns:
plot_level(codigo)
os.system(
'scp /media/nicolas/Home/Jupyter/MarioLoco/real_time/tres_horas/* [email protected]:/var/www/mario/realTime/tres_horas/'
)
os.system(
'scp /media/nicolas/Home/Jupyter/MarioLoco/real_time/tres_horas/* [email protected]:/var/www/mario/realTime/tres_horas/'
)
os.system(
'scp /media/nicolas/Home/Jupyter/MarioLoco/real_time/tres_horas/* [email protected]:/var/www/mario/realTime/tres_horas/'
)
def plot_level(codigo):
resolution = '3h' # will be argument later on
path = '/media/nicolas/Home/Jupyter/MarioLoco'
if resolution == '3h':
folder = 'tres_horas'
obj = cpr.Nivel(codigo=codigo,
user='******',
passwd='s@mple_p@ss',
SimuBasin=False)
levantamiento = pd.read_csv('%s/ultimos_levantamientos/%s.csv' %
(path, codigo),
index_col=0)
filepath = '%s/%s.png' % (path + '/real_time/' + folder, obj.info.slug)
obj.plot_operacional(df[codigo] / 100.0,
levantamiento,
resolution,
filepath=filepath)
print(codigo)
# -*- coding: utf-8 -*-
#
# CRP.py
#
# Copyright 2018 MCANO <*****@*****.**>
import cprv1.cprv1 as cpr
import datetime
import pandas as pd
import os
import matplotlib.pyplot as plt
import numpy as np
import matplotlib.dates as mdates
self = cpr.Nivel(codigo=99,
user='******',
passwd='s@mple_p@ss',
SimuBasin=True)
date = datetime.datetime.now()
fechas = []
#def reporte_diario(self,date):
end = pd.to_datetime(pd.to_datetime(date).strftime('%Y-%m-%d') +
' 23:50') - datetime.timedelta(days=1)
start = (end - datetime.timedelta(days=6)).strftime('%Y-%m-%d 00:00')
folder_path = '/media/nicolas/Home/Jupyter/MarioLoco/reporte_diario/%s' % end.strftime(
'%Y%m%d')
folder_name = end.strftime('%Y%m%d')
os.system('mkdir %s' % folder_path)
df = self.level_all(start, end, calidad=True)
from matplotlib.patches import Rectangle
end = dt.datetime.now()
starts = [
(end - dt.timedelta(hours=3)), (end - dt.timedelta(hours=24)),
(end - dt.timedelta(hours=72)), (end - dt.timedelta(days=30))
]
for start in starts:
#consulta pluvio
pluvio = cpr_1.Pluvio(int(self.info.get('pluvios')))
pluvio_s = pluvio.read_pluvio(start, end)
self.plot_Humedad2Webpage(start, end, pluvio_s, ruta_figs,
rutacredentials_remote,
rutacredentials_local)
print 'Se ejecutan las graficas operacionales de la red de humedad'
#EJECUCION
self = cpr.Nivel(codigo=99,
user='******',
passwd='s@mple_p@ss',
SimuBasin=False)
# Plots de Red de Humedad,ejecuta plots en paralelo
if __name__ == '__main__':
p = multiprocessing.Process(target=plot_HNetwork, args=(self, ), name="r")
p.start()
time.sleep(250) # wait near 5 minutes to kill process
p.terminate()
p.join()
print 'plot_HNetwork executed'
def reporte_lluvia():
try:
self = cpr.Nivel(codigo=260,
user='******',
passwd='s@mple_p@ss',
SimuBasin=True)
end = datetime.datetime.now()
start = end - datetime.timedelta(hours=3)
posterior = end + datetime.timedelta(minutes=10)
rain = self.radar_rain(start, posterior)
rain_vect = self.radar_rain_vect(start, posterior)
codigos = self.infost.index
df = pd.DataFrame(index=rain_vect.index, columns=codigos)
for codigo in codigos:
mask_path = '/media/nicolas/maso/Mario/mask/mask_%s.tif' % (codigo)
try:
mask_map = wmf.read_map_raster(mask_path)
mask_vect = self.Transform_Map2Basin(mask_map[0], mask_map[1])
except AttributeError:
print 'mask:%s' % codigo
mask_vect = None
if mask_vect is not None:
mean = []
for date in rain_vect.index:
try:
mean.append(
np.sum(mask_vect * rain_vect.loc[date]) /
np.sum(mask_vect))
except:
print 'mean:%s' % codigo
if len(mean) > 0:
df[codigo] = mean
df_posterior = df.loc[end:]
plt.rc('font', **{'size': 16})
fig = plt.figure(figsize=(20, 20))
fig.subplots_adjust(hspace=1.1)
ax1 = fig.add_subplot(211)
ax2 = fig.add_subplot(212)
suma = (df / 1000.).sum().sort_values(ascending=False)
suma = suma[suma > 0.0]
orden = np.array(suma.index, int)
suma.index = self.infost.loc[suma.index, 'nombre']
risk = convert_series_to_risk(self.level_all(hours=1).iloc[-3:].max())
dfb = pd.DataFrame(index=suma.index, columns=['rain', 'color'])
dfb['rain'] = suma.values
dfb['color'] = risk.loc[orden].values
dfb.plot.bar(y='rain', color=[dfb['color']], ax=ax1)
#suma.plot(kind='bar',color = list(),ax=ax1)
title = 'start: %s, end: %s' % (start.strftime('%Y-%m-%d %H:%M'),
end.strftime('%Y-%m-%d %H:%M'))
filepath = '/media/nicolas/Home/Jupyter/MarioLoco/reportes/lluvia_en_cuencas.png'
ax1.set_title(title)
ax1.set_ylabel('lluvia acumulada\n promedio en la cuenca [mm]')
suma = (df_posterior / 1000.).sum().loc[orden]
suma.index = self.infost.loc[suma.index, 'nombre']
dfb = pd.DataFrame(index=suma.index, columns=['rain', 'color'])
dfb['rain'] = suma.values
dfb['color'] = risk.loc[orden].values
dfb.plot.bar(y='rain', color=[dfb['color']], ax=ax2)
#suma.plot(kind='bar',ax=ax2)
filepath = '/media/nicolas/Home/Jupyter/MarioLoco/reportes/lluvia_en_cuencas.png'
ax2.set_title(u'lluvia acumulada en la próxima media hora')
ax2.set_ylabel('lluvia acumulada\n promedio en la cuenca [mm]')
ax1.set_ylim(0, 30)
ax2.set_ylim(0, 30)
plt.savefig(filepath, bbox_inches='tight')
os.system(
'scp %s [email protected]:/var/www/mario/realTime/reporte_lluvia_cuenca.png'
% filepath)
except:
pass
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2018 MCANO <*****@*****.**>
import matplotlib.dates as mdates
import cprv1.cprv1 as cpr
import pandas as pd
import numpy as np
import datetime
import time
import matplotlib.pyplot as plt
import os
self = cpr.Nivel(codigo=140,user='******',passwd='s@mple_p@ss',SimuBasin=True)
end = self.round_time(datetime.datetime.now())
start = self.round_time(end - datetime.timedelta(days=2))
df = self.level_all(start,end)
df = df.resample('5min').mean()
dfr = self.risk_df(df)
dfr[dfr<2]=np.NaN
dfr = dfr.drop(260)
for codigo in dfr.index:
try:
self = cpr.Nivel(codigo=codigo,user='******',passwd='s@mple_p@ss',SimuBasin=True)
dates = dfr.loc[codigo].dropna().index
self.radar_rain(dates[0],dates[-1])
print codigo
for date in dates:
try:
self.rain_report(date)
print date
except:
return transfer
def export_tables(self, tablas):
for tabla in tablas:
initia = datetime.datetime.now()
filename = '%s.csv' % tabla
local_path = '/media/nicolas/maso/Mario/data_migration/%s' % filename
remote_path = "[email protected]:data_migration/%s" % filename
self.read_sql('select * from %s' % tabla).to_csv(local_path)
print(os.system('scp %s %s' % (local_path, remote_path)))
print('filename:%s,took:%s' %
(filename, datetime.datetime.now() - inicia))
remote = cpr.Nivel(**cpr.info.REMOTE)
local = cpr.Nivel(codigo=99, user='******', passwd='s@mple_p@ss')
initial_run = datetime.datetime.now()
df = remote.read_sql(
"select codigo,FechaInstalacion from estaciones where red='Nivel'")
s = pd.to_datetime(df['FechaInstalacion'])
s[s < '2000-01-01'] = np.NaN
initial = pd.to_datetime("2017-11-26")
daily = pd.date_range(initial, datetime.datetime.now())
sundays = daily[daily.strftime('%a') == 'dom']
# arguments
for count in np.arange(0, len(sundays) - 1):
inicia = datetime.datetime.now()
start = sundays[count]
end = sundays[count + 1]
def update_all(self,df):
for codigo in self.infost.index:
obj = cpr.Nivel(codigo=codigo,user='******',passwd='s@mple_p@ss')
obj.table = 'hydro'
obj.update_series(df[codigo],'nivel')
def plot_acum_radar():
#DEFINICION DE COSAS
rutafig = '/media/nicolas/Home/Jupyter/Soraya/Op_Alarmas/Result_to_web/operacional/acum_radar/'
selfN = cprv1.Nivel(codigo=260,
user='******',
passwd='s@mple_p@ss',
SimuBasin=True)
codigos = selfN.infost.index[:]
#setting the est indexes and order.
pos_1st = [51, 36, 37, 40, 54, 64]
codigos = np.delete(codigos, pos_1st)
codigos = np.insert(codigos, 0, 260)
#time windows
windows_t = ['30m_ahead', '10m', '5m', '3h'] #,'6h','24h','3d']
#PLOTS
for window_t in windows_t[:2]:
print window_t
if window_t == '30m_ahead':
start = pd.to_datetime('2019-05-09 22:00')
# start= fs.round_time(dt.datetime.now())
end = start + pd.Timedelta('30m')
else:
end = pd.to_datetime('2019-05-09 22:00')
# end = fs.round_time(dt.datetime.now())
start = end - pd.Timedelta(window_t)
path_basins = '/media/nicolas/maso/Mario/basins/'
#acumulado para la cuenca mas grande
path_radtif = '/media/nicolas/maso/Soraya/op_files/radar/tifs/260-' + window_t + '.tif'
dfAll, rvec = fs.get_radar_rain(
start,
end,
300.,
path_basins + '%s.nc' % (260),
codigos,
accum=True,
path_tif=
path_radtif, #+start.strftime('%Y%m%d%H%M')+'_'+start.strftime('%Y%m%d%H%M')+'.tif',
meanrain_ALL=True)
dfAll.to_csv(rutafig + window_t + '/dfAcum' + window_t + '.csv')
#plots
for codigo in codigos[0:]:
if '%s.nc' % (codigo) in os.listdir(path_basins):
cu = wmf.SimuBasin(rute=path_basins + '%s.nc' % (codigo))
a, b = wmf.read_map_raster(path_radtif)
vec_rain = cu.Transform_Map2Basin(a, b)
#Plot
fig = pl.figure(figsize=(10, 12))
ax = fig.add_subplot(111)
if window_t == '30m_ahead':
fs.plot_basin_rain(cu,
vec_rain,
codigo,
window_t='30m',
ax=ax,
cbar=True)
else:
fs.plot_basin_rain(cu,
vec_rain,
codigo,
window_t='30m',
ax=ax,
cbar=True)
ax.set_title('Est. ' + str(codigo) + ' | ' +
selfN.infost.loc[codigo].nombre + '\n' +
pd.to_datetime(start).strftime('%Y%m%d%H%M') +
'-' + pd.to_datetime(end).strftime('%Y%m%d%H%M'))
pl.savefig(rutafig + window_t + '/' +
selfN.infost.loc[codigo].slug + '.png',
bbox_inches='tight')
