def siata_remote_data_to_transfer(start, end, *args, **kwargs): remote = cpr.Nivel(**cpr.info.REMOTE) codigos_str = '(' + str(list(local.infost.index)).strip('[]') + ')' parameters = tuple([codigos_str, local.fecha_hora_query(start, end)]) df = remote.read_sql('SELECT * FROM datos WHERE cliente in %s and %s' % parameters) return df
def plot_level(serie,codigo,resolution,folder,path = '/media/nicolas/Home/Jupyter/MarioLoco'): try: # will be argument later on obj = cpr.Nivel(codigo = codigo,user='******',passwd='s@mple_p@ss',SimuBasin=False) levantamiento = pd.read_csv('%s/ultimos_levantamientos/%s.csv'%(path,codigo),index_col=0) filepath = '%s/%s.png'%(path+'/real_time/'+folder,obj.info.slug) obj.plot_operacional(serie,levantamiento,resolution,filepath=filepath) r = os.system('scp %s [email protected]:/var/www/mario/realTime/%s'%(filepath,folder)) except: print 'error in plot %s'%codigo
def cron(): def convert_to_risk(df): df = self.risk_df(df) return df[df.columns.dropna()] #PROCESS self = cpr.Nivel(codigo=99, user='******', passwd='s@mple_p@ss', SimuBasin=False) end = datetime.datetime.now() start = end - datetime.timedelta(minutes=150) df = self.level_all(start, end, calidad=True) risk_df = convert_to_risk(df.copy()) def select(pos): try: colors = ['green', '#F7DD1E', '#F9A43E', '#FC3A3A', '#FC3A3A'] return colors[pos] except IndexError: return '#D8E0E8' filepath = "http://siata.gov.co/mario/realTime/tres_horas/" data_list = [] for codigo in risk_df.index[::-1]: positions = np.array(risk_df.loc[codigo].values, int) color = map(lambda x: select(x), positions) data = pd.DataFrame(color, index=risk_df.loc[codigo].index) data['hour'] = risk_df.loc[codigo].index.strftime('%H:%M') data['nombre'] = "%s | " % codigo + str(self.infost.loc[codigo, 'nombre']) data['path'] = filepath + self.infost.loc[codigo, 'slug'] + '.png' data['location'] = self.infost.loc[codigo, 'municipio'] data['telefono'] = 'No' data['celular'] = 'No' data['waterlevel'] = df.T.loc[codigo] data.columns = [ 'color', 'hour', 'name', 'path', 'location', 'phone', 'mobile', 'waterlevel' ] data_list.append(data) data_list = pd.concat(data_list) data_list.name = data_list.name.str.replace(' - Nivel', '') filename = 'heatmap_data.csv' data_list.to_csv(filename) statement = 'scp %s [email protected]:/var/www/mario/realTime/risk_levels_chart/%s' % ( filename, filename) print(os.system(statement)) print(os.system(statement)) print(os.system(statement))
def convert_series_to_risk(level): '''level: pandas Series, index = codigos de estaciones''' risk = level.copy() colors = ['green', 'gold', 'orange', 'red', 'red', 'black'] for codigo in level.index: try: risks = cpr.Nivel(codigo=codigo, user='******', passwd='s@mple_p@ss').risk_levels risk[codigo] = colors[int( self.convert_level_to_risk(level[codigo], risks))] except: risk[codigo] = 'black' return risk
def process_multiple_plots_looping(): self = cpr.Nivel(codigo=99, user='******', passwd='s@mple_p@ss', SimuBasin=False) for codigo in df.columns: plot_level(codigo) os.system( 'scp /media/nicolas/Home/Jupyter/MarioLoco/real_time/tres_horas/* [email protected]:/var/www/mario/realTime/tres_horas/' ) os.system( 'scp /media/nicolas/Home/Jupyter/MarioLoco/real_time/tres_horas/* [email protected]:/var/www/mario/realTime/tres_horas/' ) os.system( 'scp /media/nicolas/Home/Jupyter/MarioLoco/real_time/tres_horas/* [email protected]:/var/www/mario/realTime/tres_horas/' )
def plot_level(codigo): resolution = '3h' # will be argument later on path = '/media/nicolas/Home/Jupyter/MarioLoco' if resolution == '3h': folder = 'tres_horas' obj = cpr.Nivel(codigo=codigo, user='******', passwd='s@mple_p@ss', SimuBasin=False) levantamiento = pd.read_csv('%s/ultimos_levantamientos/%s.csv' % (path, codigo), index_col=0) filepath = '%s/%s.png' % (path + '/real_time/' + folder, obj.info.slug) obj.plot_operacional(df[codigo] / 100.0, levantamiento, resolution, filepath=filepath) print(codigo)
# -*- coding: utf-8 -*- # # CRP.py # # Copyright 2018 MCANO <*****@*****.**> import cprv1.cprv1 as cpr import datetime import pandas as pd import os import matplotlib.pyplot as plt import numpy as np import matplotlib.dates as mdates self = cpr.Nivel(codigo=99, user='******', passwd='s@mple_p@ss', SimuBasin=True) date = datetime.datetime.now() fechas = [] #def reporte_diario(self,date): end = pd.to_datetime(pd.to_datetime(date).strftime('%Y-%m-%d') + ' 23:50') - datetime.timedelta(days=1) start = (end - datetime.timedelta(days=6)).strftime('%Y-%m-%d 00:00') folder_path = '/media/nicolas/Home/Jupyter/MarioLoco/reporte_diario/%s' % end.strftime( '%Y%m%d') folder_name = end.strftime('%Y%m%d') os.system('mkdir %s' % folder_path) df = self.level_all(start, end, calidad=True) from matplotlib.patches import Rectangle
end = dt.datetime.now() starts = [ (end - dt.timedelta(hours=3)), (end - dt.timedelta(hours=24)), (end - dt.timedelta(hours=72)), (end - dt.timedelta(days=30)) ] for start in starts: #consulta pluvio pluvio = cpr_1.Pluvio(int(self.info.get('pluvios'))) pluvio_s = pluvio.read_pluvio(start, end) self.plot_Humedad2Webpage(start, end, pluvio_s, ruta_figs, rutacredentials_remote, rutacredentials_local) print 'Se ejecutan las graficas operacionales de la red de humedad' #EJECUCION self = cpr.Nivel(codigo=99, user='******', passwd='s@mple_p@ss', SimuBasin=False) # Plots de Red de Humedad,ejecuta plots en paralelo if __name__ == '__main__': p = multiprocessing.Process(target=plot_HNetwork, args=(self, ), name="r") p.start() time.sleep(250) # wait near 5 minutes to kill process p.terminate() p.join() print 'plot_HNetwork executed'
def reporte_lluvia(): try: self = cpr.Nivel(codigo=260, user='******', passwd='s@mple_p@ss', SimuBasin=True) end = datetime.datetime.now() start = end - datetime.timedelta(hours=3) posterior = end + datetime.timedelta(minutes=10) rain = self.radar_rain(start, posterior) rain_vect = self.radar_rain_vect(start, posterior) codigos = self.infost.index df = pd.DataFrame(index=rain_vect.index, columns=codigos) for codigo in codigos: mask_path = '/media/nicolas/maso/Mario/mask/mask_%s.tif' % (codigo) try: mask_map = wmf.read_map_raster(mask_path) mask_vect = self.Transform_Map2Basin(mask_map[0], mask_map[1]) except AttributeError: print 'mask:%s' % codigo mask_vect = None if mask_vect is not None: mean = [] for date in rain_vect.index: try: mean.append( np.sum(mask_vect * rain_vect.loc[date]) / np.sum(mask_vect)) except: print 'mean:%s' % codigo if len(mean) > 0: df[codigo] = mean df_posterior = df.loc[end:] plt.rc('font', **{'size': 16}) fig = plt.figure(figsize=(20, 20)) fig.subplots_adjust(hspace=1.1) ax1 = fig.add_subplot(211) ax2 = fig.add_subplot(212) suma = (df / 1000.).sum().sort_values(ascending=False) suma = suma[suma > 0.0] orden = np.array(suma.index, int) suma.index = self.infost.loc[suma.index, 'nombre'] risk = convert_series_to_risk(self.level_all(hours=1).iloc[-3:].max()) dfb = pd.DataFrame(index=suma.index, columns=['rain', 'color']) dfb['rain'] = suma.values dfb['color'] = risk.loc[orden].values dfb.plot.bar(y='rain', color=[dfb['color']], ax=ax1) #suma.plot(kind='bar',color = list(),ax=ax1) title = 'start: %s, end: %s' % (start.strftime('%Y-%m-%d %H:%M'), end.strftime('%Y-%m-%d %H:%M')) filepath = '/media/nicolas/Home/Jupyter/MarioLoco/reportes/lluvia_en_cuencas.png' ax1.set_title(title) ax1.set_ylabel('lluvia acumulada\n promedio en la cuenca [mm]') suma = (df_posterior / 1000.).sum().loc[orden] suma.index = self.infost.loc[suma.index, 'nombre'] dfb = pd.DataFrame(index=suma.index, columns=['rain', 'color']) dfb['rain'] = suma.values dfb['color'] = risk.loc[orden].values dfb.plot.bar(y='rain', color=[dfb['color']], ax=ax2) #suma.plot(kind='bar',ax=ax2) filepath = '/media/nicolas/Home/Jupyter/MarioLoco/reportes/lluvia_en_cuencas.png' ax2.set_title(u'lluvia acumulada en la próxima media hora') ax2.set_ylabel('lluvia acumulada\n promedio en la cuenca [mm]') ax1.set_ylim(0, 30) ax2.set_ylim(0, 30) plt.savefig(filepath, bbox_inches='tight') os.system( 'scp %s [email protected]:/var/www/mario/realTime/reporte_lluvia_cuenca.png' % filepath) except: pass
#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2018 MCANO <*****@*****.**> import matplotlib.dates as mdates import cprv1.cprv1 as cpr import pandas as pd import numpy as np import datetime import time import matplotlib.pyplot as plt import os self = cpr.Nivel(codigo=140,user='******',passwd='s@mple_p@ss',SimuBasin=True) end = self.round_time(datetime.datetime.now()) start = self.round_time(end - datetime.timedelta(days=2)) df = self.level_all(start,end) df = df.resample('5min').mean() dfr = self.risk_df(df) dfr[dfr<2]=np.NaN dfr = dfr.drop(260) for codigo in dfr.index: try: self = cpr.Nivel(codigo=codigo,user='******',passwd='s@mple_p@ss',SimuBasin=True) dates = dfr.loc[codigo].dropna().index self.radar_rain(dates[0],dates[-1]) print codigo for date in dates: try: self.rain_report(date) print date except:
return transfer def export_tables(self, tablas): for tabla in tablas: initia = datetime.datetime.now() filename = '%s.csv' % tabla local_path = '/media/nicolas/maso/Mario/data_migration/%s' % filename remote_path = "[email protected]:data_migration/%s" % filename self.read_sql('select * from %s' % tabla).to_csv(local_path) print(os.system('scp %s %s' % (local_path, remote_path))) print('filename:%s,took:%s' % (filename, datetime.datetime.now() - inicia)) remote = cpr.Nivel(**cpr.info.REMOTE) local = cpr.Nivel(codigo=99, user='******', passwd='s@mple_p@ss') initial_run = datetime.datetime.now() df = remote.read_sql( "select codigo,FechaInstalacion from estaciones where red='Nivel'") s = pd.to_datetime(df['FechaInstalacion']) s[s < '2000-01-01'] = np.NaN initial = pd.to_datetime("2017-11-26") daily = pd.date_range(initial, datetime.datetime.now()) sundays = daily[daily.strftime('%a') == 'dom'] # arguments for count in np.arange(0, len(sundays) - 1): inicia = datetime.datetime.now() start = sundays[count] end = sundays[count + 1]
def update_all(self,df): for codigo in self.infost.index: obj = cpr.Nivel(codigo=codigo,user='******',passwd='s@mple_p@ss') obj.table = 'hydro' obj.update_series(df[codigo],'nivel')
def plot_acum_radar(): #DEFINICION DE COSAS rutafig = '/media/nicolas/Home/Jupyter/Soraya/Op_Alarmas/Result_to_web/operacional/acum_radar/' selfN = cprv1.Nivel(codigo=260, user='******', passwd='s@mple_p@ss', SimuBasin=True) codigos = selfN.infost.index[:] #setting the est indexes and order. pos_1st = [51, 36, 37, 40, 54, 64] codigos = np.delete(codigos, pos_1st) codigos = np.insert(codigos, 0, 260) #time windows windows_t = ['30m_ahead', '10m', '5m', '3h'] #,'6h','24h','3d'] #PLOTS for window_t in windows_t[:2]: print window_t if window_t == '30m_ahead': start = pd.to_datetime('2019-05-09 22:00') # start= fs.round_time(dt.datetime.now()) end = start + pd.Timedelta('30m') else: end = pd.to_datetime('2019-05-09 22:00') # end = fs.round_time(dt.datetime.now()) start = end - pd.Timedelta(window_t) path_basins = '/media/nicolas/maso/Mario/basins/' #acumulado para la cuenca mas grande path_radtif = '/media/nicolas/maso/Soraya/op_files/radar/tifs/260-' + window_t + '.tif' dfAll, rvec = fs.get_radar_rain( start, end, 300., path_basins + '%s.nc' % (260), codigos, accum=True, path_tif= path_radtif, #+start.strftime('%Y%m%d%H%M')+'_'+start.strftime('%Y%m%d%H%M')+'.tif', meanrain_ALL=True) dfAll.to_csv(rutafig + window_t + '/dfAcum' + window_t + '.csv') #plots for codigo in codigos[0:]: if '%s.nc' % (codigo) in os.listdir(path_basins): cu = wmf.SimuBasin(rute=path_basins + '%s.nc' % (codigo)) a, b = wmf.read_map_raster(path_radtif) vec_rain = cu.Transform_Map2Basin(a, b) #Plot fig = pl.figure(figsize=(10, 12)) ax = fig.add_subplot(111) if window_t == '30m_ahead': fs.plot_basin_rain(cu, vec_rain, codigo, window_t='30m', ax=ax, cbar=True) else: fs.plot_basin_rain(cu, vec_rain, codigo, window_t='30m', ax=ax, cbar=True) ax.set_title('Est. ' + str(codigo) + ' | ' + selfN.infost.loc[codigo].nombre + '\n' + pd.to_datetime(start).strftime('%Y%m%d%H%M') + '-' + pd.to_datetime(end).strftime('%Y%m%d%H%M')) pl.savefig(rutafig + window_t + '/' + selfN.infost.loc[codigo].slug + '.png', bbox_inches='tight')