def SQL_Read_MeanRainfall(link_id,
date1,
date2,
schema='pers_nico',
table='s4mrain',
time_name='unix_time',
data_name='rain',
linkid_name='link_id'):
'''DEPRECATED Read streamflow data from IIHR database "research_environment"
and returns it as a pandas.DataFrame element.
Parameters:
- usgs_id: code of the usgs.
- date1: initial date of the query.
- date2: final date of the query.
Optional:
- schema: where to obtain data in the databse.
- table: master table with the usgs data.
- time_name: the name of the column that has the time.
- data_name: the name of the column that has the data.
- usgs_name: the name of the column that has the id of the usgs stations.
Returns:
- pandas.DataFrame containing the streamflow data.'''
#make the connection
con = DataBaseConnect(user='******', password='******')
#Work with dates and usgs id
date1 = str(aux.__datetime2unix__(date1))
date2 = str(aux.__datetime2unix__(date2))
if type(link_id) is not str:
link_id = str(link_id)
#make the querty
query = sql.SQL("SELECT " + time_name + ", " + data_name + " FROM " +
schema + "." + table + " WHERE " + time_name +
" BETWEEN " + date1 + " and " + date2 + " AND " +
linkid_name + "='" + link_id + "'")
#Make the consult.
Data = pd.read_sql(query,
con,
index_col='unix_time',
parse_dates={'unix_time': {
'unit': 's'
}})
con.close()
#Organize rainfall
Data = Data.sort_index()
Dates = pd.date_range(Data.index[0], Data.index[-1], freq='1h')
Rain = pd.Series(np.zeros(Dates.size), Dates)
Rain[Data.index] = Data['rain'].values
Rain[Rain > 1000] = 0.0
return Rain
def UpdateGlobals(self, dt, initial_old, initial_new):
#Actual date
date_old = self.dateEvent
date_new = self.dateEvent + pd.Timedelta(dt)
date_fut = date_new + pd.Timedelta(dt)
#Get old and new dates
date_old_t = date_old.strftime('%Y-%m-%d %H:%M')
date_new_t = date_new.strftime('%Y-%m-%d %H:%M')
date_fut_t = date_fut.strftime('%Y-%m-%d %H:%M')
#Unix times to replace
unix1 = str(aux.__datetime2unix__(date_old_t) + 12 * 3600)
unix2 = str(aux.__datetime2unix__(date_new_t) + 12 * 3600)
unix3 = str(aux.__datetime2unix__(date_fut_t) + 12 * 3600)
#Creates the dictionary with the keys to replace
D = {
'date_f': {
'old': date_new_t,
'new': date_fut_t
},
'date_i': {
'old': date_old_t,
'new': date_new_t
},
'initial': {
'old': initial_old,
'new': initial_new
},
'unix_f': {
'old': unix2,
'new': unix3
},
'unix_i': {
'old': unix1,
'new': unix2
}
}
#Creates the list for updates of the globals
Lista = []
for k in self.setups.keys():
Lista.append([self.path + '/ForRun/' + k + '.gbl', D])
#Update globals with a multiprocessing approach
self.execute(WarpUpdateGlobals, Lista)
self.dateEvent = date_new
def SQL_read_USGS_Streamflow(usgs_id,
date1,
date2,
schema='pers_nico',
table='data_usgs',
time_name='unix_time',
data_name='val',
usgs_name='usgs_id'):
'''Read streamflow data from IIHR database "research_environment"
and returns it as a pandas.DataFrame element.
Parameters:
- usgs_id: code of the usgs.
- date1: initial date of the query.
- date2: final date of the query.
Optional:
- schema: where to obtain data in the databse.
- table: master table with the usgs data.
- time_name: the name of the column that has the time.
- data_name: the name of the column that has the data.
- usgs_name: the name of the column that has the id of the usgs stations.
Returns:
- pandas.DataFrame containing the streamflow data.'''
#make the connection
con = DataBaseConnect(user='******', password='******')
#Work with dates and usgs id
date1 = str(aux.__datetime2unix__(date1))
date2 = str(aux.__datetime2unix__(date2))
if type(usgs_id) is not str:
usgs_id = str(usgs_id)
#make the querty
query = sql.SQL("SELECT " + time_name + ", " + data_name + " FROM " +
schema + "." + table + " WHERE " + time_name +
" BETWEEN " + date1 + " and " + date2 + " AND " +
usgs_name + "='" + usgs_id + "'")
#Make the consult.
Data = pd.read_sql(query,
con,
index_col='unix_time',
parse_dates={'unix_time': {
'unit': 's'
}})
con.close()
return Data
def write_rainfall(self, date1, date2, path):
'''Writes binary files for the rainfall of the defined watershed.
- date1: initial date of the data to write.
- date2: end date
- path: where to store binary files eg: /home/user/basin/rain/BasinStage4_
Returns:
- the mean rainfall of the watershed for the selected period'''
#Databse connection and query.
unix1 = str(aux.__datetime2unix__(date1))
unix2 = str(aux.__datetime2unix__(date2))
con = db.DataBaseConnect(database='rt_precipitation')
if self.linkid > 0:
q = db.sql.SQL("WITH subbasin AS (SELECT nodeX.link_id AS link_id FROM students.env_master_km \
AS nodeX, students.env_master_km AS parentX WHERE (nodeX.left BETWEEN parentX.left AND parentX.right) \
AND parentX.link_id = "+str(self.linkid)+") SELECT A.unix_time,sum(weight*A.val) as rain,B.link_id FROM stage_4.data AS \
A,env_lookup_hrap_lid_v4 AS B,subbasin WHERE A.grid_x = B.x AND A.grid_y=B.y AND B.link_id = subbasin.link_id \
AND A.unix_time >= "+str(unix1)+" AND A.unix_time < "+str(unix2)+" AND A.val < 99.0 GROUP BY B.link_id,A.unix_time ORDER BY A.unix_time")
else:
q = db.sql.SQL("SELECT \
A.unix_time, \
sum(weight*A.val) as rain, \
B.link_id \
FROM \
stage_4.data AS A, \
env_lookup_hrap_lid_v4 AS B \
WHERE A.grid_x = B.x AND A.grid_y=B.y AND A.unix_time >= "+str(unix1)+" AND A.unix_time < "+str(unix2)+" AND A.val < 999.0 \
GROUP BY B.link_id,A.unix_time \
ORDER BY A.unix_time;")
E = pd.read_sql(q, con, index_col='unix_time')
con.close()
#SEtup for the data with the mean rainfall for that period
d = pd.date_range(date1, date2, freq='1H')
MeanRain = pd.Series(np.zeros(d.size), index = d)
#Rainfall binary files creation
for i in np.arange(E.index[0], E.index[-1], 3600):
Rain = E[['link_id','rain']][E.index == i]
__saveBin__(Rain['link_id'].values, Rain['rain'].values, Rain['rain'].size,path+str(i))
MeanRain[pd.to_datetime(i,unit='s')] = Rain['rain'].mean()
MeanRain[np.isnan(MeanRain) == True] = 0.0
return MeanRain
def SQL_Get_MeanRainfall(linkID, date1, date2):
'''Obtains the mean rainfall for the watershed associated to
a given linkID.
Parameters:
- linkID: linkID of the outlet of the basin.
- date1: initial date (YYYY-MM-DD HH:MM).
- date2: end date (YYYY-MM-DD HH:MM).
Returns:
- Rainfall: Pandas series with the mean rainfall in the basin.'''
#SEt the connection
con = DataBaseConnect(user='******',
password='******',
database='rt_precipitation')
#Transform dates to unix
unix1 = str(aux.__datetime2unix__(date1))
unix2 = str(aux.__datetime2unix__(date2))
linkID = str(linkID)
#Set the query and obtains data
q = sql.SQL(
"WITH subbasin AS (SELECT nodeX.link_id AS link_id FROM students.env_master_km AS nodeX, students.env_master_km AS parentX WHERE (nodeX.left BETWEEN parentX.left AND parentX.right) AND parentX.link_id = "
+ str(linkID) +
"), uparea as (SELECT up_area FROM students.env_master_km WHERE link_id= "
+ str(linkID) +
"), lut as (SELECT x, y FROM env_lookup_hrap_lid_v4 WHERE link_id IN (SELECT * FROM subbasin) group by x, y) SELECT unix_time, sum(val)/(SELECT count(*) FROM lut) as rain FROM stage_4.data WHERE grid_x IN (SELECT x FROM lut) AND grid_y IN (SELECT y from lut) AND unix_time between "
+ unix1 + " AND " + unix2 + " group by unix_time order by unix_time;")
Data = pd.read_sql(q,
con,
index_col='unix_time',
parse_dates={'unix_time': {
'unit': 's'
}})
#close connection
con.close()
#Pos process data
dates = pd.date_range(date1, date2, freq='1h')
Rain = pd.Series(np.zeros(dates.size), dates)
Rain[Data.index] = Data['rain']
return Rain
fl.Evento.CreateBashFile(status='start',
path='./Run_' + str(args.link) + '_' +
dateText + '_' + lambdaName + '.sh')
#First initial condition
Initial = fl.Evento.path + '/ForRun/initial.dbc'
initFlag = '3'
#Creates the gbl files for each epoc
first = 'si'
for d1, d2 in zip(Dates[:-1], Dates[1:]):
#Dates in string format
d_initial = d1.strftime('%Y-%m-%d %H:%M')
d_end = d2.strftime('%Y-%m-%d %H:%M')
#Creates the list for execution
unix1 = str(aux.__datetime2unix__(d_initial)) #+12*3600)
unix2 = str(aux.__datetime2unix__(d_end)) #+12*3600)
#Iterates
Lista = []
for c, rc in enumerate(args.rc):
#Name of the project
name = d1.strftime('%Y%m%d%H%M') + '_' + str(
c) + '_' + lambdaName
#Parameters
params = '6 0.75 %s -0.2 %s 0.1 2.2917e-05' % (lamb, str(rc))
#List for that run
L = [
d_initial, d_end, params, args.link, initFlag, Initial,
unix1, unix2, fl.Evento.path + '/Results/' + name + '.dat',
fl.Evento.path + '/ForRun/control.sav',
fl.Evento.path + '/Initial/' + name + '.h5',
def ASYNCH_setGlobal(self, gblBase = 'BaseGlobal.gbl', Links2SaveName = 'ControlPoints.sav',
OutStatesName = 'OutputStates.dat', initial_name = None, initial_exist = False,
snapName = None, snapTime = None, createSav = False):
'''Edit the global file for asynch run.
Parameters:
- date1: the initial date of the simulation (YYYY-MM-DD HH:MM)
- date2: the initial date of the simulation (YYYY-MM-DD HH:MM)
- linkID: the number of the output link to make the simulation.
- glbOut: rute and name of the output global file for asynch.
- output: name of the file with the outputs.
- peakflow: name of the file containing the links where to save.
Optional:
- parameters: running parameters for ASYNCH model.
- unix1: start time of the execution.
- unix2: end time of the execution.
- glbBase: rute and name of the base globl file used for the excecutions.
Outputs:
This function writes a gbl file where gblOut indicates.'''
# Copy the base global to a glbOut
self.path_in_global = self.path_in + self.name + '.gbl'
comand = 'cp '+Path+self.model+gblBase+' '+self.path_in_global
os.system(comand)
#Copy the links2save file
if createSav is False:
self.path_in_links2save = self.path_in + Links2SaveName
comand = 'cp '+Path+self.links2save+' '+self.path_in_links2save
os.system(comand)
else:
self.path_in_links2save = self.path_in+'control.sav'
f = open(self.path_in_links2save,'w')
f.write('%s' % self.linkID)
f.close()
#Set of the initial file for that link
if initial_name is None:
self.path_in_initial = self.path_in + self.name + '.dbc'
else:
self.path_in_initial = initial_name
if initial_exist is False:
self.__ASYNCH_setInitialFile__(self.path_in_initial,self.date1[:4],
self.linkID)
#Set the number of the initial depending on its extension
InitNumber = str(self.__ASYNCH_get_number(self.path_in_initial, whatfor='initial'))
#Set the snapshot info
if snapName is not None:
Snap_flag = self.__ASYNCH_get_number(snapName, whatfor='snapshot')
Snap_name = snapName
if snapTime is not None:
Snap_time = str(snapTime)
if Snap_flag == 3:
Snap_flag = str(4)
else:
Snap_time = ''
Snap_flag = str(Snap_flag)
else:
Snap_flag = 0; Snap_name = ''; Snap_time = ''
#Set the name of the file with the output of the streamflow
self.path_out_states = self.path_out + OutStatesName
# Unix time are equal to date
if self.unix1 is None:
self.unix1 = aux.__datetime2unix__(self.date1) + 12*3600.
textUnix1 = '%d' % self.unix1
if self.unix2 is None:
self.unix2 =aux.__datetime2unix__(self.date2) + 12*3600
textUnix2 = '%d' % self.unix2
# Parameters
Param = ' '.join(self.parameters)+'\n'
# Replace parameters in the global file
DicToreplace = {'date1':{'to_search': '¿date1?', 'to_put': self.date1},
'date2':{'to_search': '¿date2?', 'to_put': self.date2},
'unix1':{'to_search': '¿unix1?', 'to_put': textUnix1},
'unix2':{'to_search': '¿unix2?', 'to_put': textUnix2},
'linkID':{'to_search': '¿linkID?', 'to_put': self.linkID},
'parameters':{'to_search': '¿Parameters?', 'to_put': Param},
'output':{'to_search': '¿output?', 'to_put': self.path_out_states},
'peakflow':{'to_search': '¿peakflow?', 'to_put': self.path_in_links2save},
'initial':{'to_search': '¿initial?', 'to_put': self.path_in_initial},
'initial_flag': {'to_search': '¿initialflag?', 'to_put': InitNumber},
'snapshot_flag': {'to_search': '¿snapflag?', 'to_put': Snap_flag},
'snapshot_time': {'to_search': '¿snaptime?', 'to_put': Snap_time},
'snapthot_name': {'to_search': '¿snapshot?', 'to_put': Snap_name},}
# Replacement in the document.
filename = self.path_in_global
for k in DicToreplace:
with fileinput.FileInput(filename, inplace=True) as file:
for line in file:
text_to_search = DicToreplace[k]['to_search']
replacement_text = str(DicToreplace[k]['to_put'])
print(line.replace(text_to_search, replacement_text), end='')
def write_Global(self, path2global, model_uid = 604,
date1 = None, date2 = None, rvrFile = None, rvrType = 0, rvrLink = 0, prmFile = None, prmType = 0, initialFile = None,
initialType = 1,rainType = 5, rainPath = None, evpFile = 'evap.mon', datResults = None,
nComponents = 1, Components = [0], controlFile = None, baseGlobal = None, noWarning = False, snapType = 0,
snapPath = '', snapTime = '', evpFromSysPath = False):
'''Creates a global file for the current project.
- model_uid: is the number of hte model goes from 601 to 604.
- date1 and date2: initial date and end date
- rvrFile: path to rvr file.
- rvrType: 0: .rvr file, 1: databse .dbc file.
- rvrLink: 0: all the domain, N: number of the linkid.
- prmFile: path to prm file.
- prmType: 0: .prm file, 1: databse .dbc file.
- initialFile: path to file with initial conditions.
- initialType: type of initial file:
- 0: ini, 1: uini, 2: rec, 3: .dbc
- rainType: number inficating the type of the rain to be used.
- 1: plain text with rainfall data for each link.
- 3: Database.
- 4: Uniform storm file: .ustr
- 5: Binary data with the unix time
- rainPath: path to the folder containning the binary files of the rain.
or path to the file with the dabase
- evpFile: path to the file with the values of the evp.
- datResults: File where .dat files will be written.
- nComponents: Number of results to put in the .dat file.
- Components: Number of each component to write: [0,1,2,...,N]
- controlFile: File with the number of the links to write.
- baseGlobal: give the option to use a base global that is not the default
- snapType: type of snapshot to make with the model:
- 0: no snapshot, 1: .rec file, 2: to database, 3: to hdf5, 4:
recurrent hdf5
- snapPath: path to the snapshot.
- snapTime: time interval between snapshots (min)
- evpFromSysPath: add the path of the system to the evp file.'''
#Open the base global file and creates tyhe template
if baseGlobal is not None:
f = open(baseGlobal, 'r')
L = f.readlines()
f.close()
else:
L = Globals['60X']
t = []
for i in L:
t += i
Base = Template(''.join(t))
# Databse rainfall
if rainType == 3 and rainPath is None:
rainPath = '/Dedicated/IFC/model_eval/forcing_rain51_5435_s4.dbc'
if rvrType == 1 and rvrFile is None:
rvrFile = '/Dedicated/IFC/model_eval/topo51.dbc'
#Chang the evp path
if evpFromSysPath:
evpFile = Path + evpFile
# Creates the default Dictionary.
Default = {
'model_uid' : model_uid,
'date1': date1,
'date2': date2,
'rvrFile': rvrFile,
'rvrType': str(rvrType),
'rvrLink': str(rvrLink),
'prmFile': prmFile,
'prmType': str(prmType),
'initialFile': initialFile,
'initialType': initialType,
'rainType': str(rainType),
'rainPath': rainPath,
'evpFile': evpFile,
'datResults': datResults,
'controlFile': controlFile,
'snapType': str(snapType),
'snapPath': snapPath,
'snapTime': str(snapTime),
'nComp': str(nComponents)
}
if date1 is not None:
Default.update({'unix1': aux.__datetime2unix__(Default['date1'])})
else:
Default.update({'unix1': '$'+'unix1'})
if date2 is not None:
Default.update({'unix2': aux.__datetime2unix__(Default['date2'])})
else:
Default.update({'unix2': '$'+'unix2'})
#Update the list of components to write
for n, c in enumerate(Components):
Default.update({'Comp'+str(n): 'State'+str(c)})
if nComponents <= 9:
for c in range(9-nComponents):
Default.update({'Comp'+str(8-c): 'XXXXX'})
#Check for parameters left undefined
D = {}
for k in Default.keys():
if Default[k] is not None:
D.update({k: Default[k]})
else:
if noWarning:
print('Warning: parameter ' + k +' left undefined model wont run')
D.update({k: '$'+k})
#Update parameter on the base and write global
f = open(path2global,'w', newline='\n')
f.writelines(Base.substitute(D))
f.close()
#Erase unused print components
f = open(path2global,'r')
L = f.readlines()
f.close()
flag = True
while flag:
try:
L.remove('XXXXX\n')
except:
flag = False
f = open(path2global,'w', newline='\n')
f.writelines(L)
f.close()
