def Dates_ampm224h(self, Dates, Hours=None, Date_Format=None):
'''
DESCRIPTION:
Converts dates from AM/PM to 24 hour dates.
_______________________________________________________________________
INPUT:
+ Dates: Vector with Dates in string format.
+ DateE: Vector with Hours in string format, defaulted to None if
the Dates vector has the dates.
_______________________________________________________________________
OUTPUT:
- DatesP: Complete Python datetime vector in 24 hour format.
'''
if Hours != None:
Hours2 = []
if Hours[0][-1] == 'a' or Hours[0][-1] == 'p':
for H in Hours:
Hours2.append(H + 'm')
else:
Hours2 = Hours
DatesC = np.array(
[i + ' ' + Hours2[ii] for ii, i in enumerate(Dates)])
else:
DatesC = []
if Dates[0][-1] == 'a' or Dates[0][-1] == 'p':
for D in Dates:
DatesC.append(D + 'm')
else:
DatesC = Dates
DatesC = np.array(DatesC)
# -------------------------
# Date_Format Verification
# -------------------------
if Date_Format == None:
Date_Formats = self.DateTimeAMPM_Formats
else:
Date_Formats = [Date_Format]
# -------------------------
# Transformation
# -------------------------
for iF, F in enumerate(Date_Formats):
try:
DatesP = np.array([datetime.strptime(i, F) for i in DatesC])
break
except:
DatesP = None
if iF == len(Date_Formats) - 1:
Er = utl.ShowError('Dates_ampm224h', 'DatesUtil',
'Bad date format, change format')
return Er
else:
continue
return DatesP
def __init__(self, PathImg='', Fol='201303060940', V=['Prec', 'Pres']):
MapFold = np.array(utl.GetFolders(PathImg + 'Maps/'))
SeriesFold = np.array(utl.GetFolders(PathImg + 'Series/'))
RadarFold = np.array(utl.GetFolders(PathImg + 'Radar/'))
xFolMap = np.where(MapFold == Fol)[0]
xFolSer = np.where(SeriesFold == Fol)[0]
xFolRad = np.where(RadarFold == Fol)[0]
if len(xFolSer) == 0:
E = utl.ShowError(
'__init__', 'MapSeriesGen',
'No se encuentra la carpeta en los dos directorios')
raise E
if len(xFolMap) != 0:
self.PathMaps = PathImg + 'Maps/' + MapFold[xFolMap[0]] + '/'
self.ArchMap = gl.glob(self.PathMaps + '*.png')
self.Names = [i[len(self.PathMaps):-4] for i in self.ArchMap]
self.PathSeries = PathImg + 'Series/' + SeriesFold[
xFolSer[0]] + '/' + V[1] + '/'
self.ArchSeries = gl.glob(self.PathSeries + '*.png')
self.NamesSeries = [
i[len(self.PathSeries):-4] for i in self.ArchSeries
]
if len(xFolRad) != 0:
# Se cargan los archivos, tener en cuenta la variable del radar 'DBZH'
self.PathRadar1 = PathImg + 'Radar/' + SeriesFold[
xFolSer[0]] + '/' + V[0] + '/DBZH/'
self.PathRadar2 = PathImg + 'Radar/' + SeriesFold[
xFolSer[0]] + '/' + V[1] + '/DBZH/'
self.ArchRadar1 = gl.glob(self.PathRadar1 + '*.png')
self.ArchRadar2 = gl.glob(self.PathRadar2 + '*.png')
self.NamesRadar1 = [
i[len(self.PathRadar1):-4] for i in self.ArchRadar1
]
self.NamesRadar2 = [
i[len(self.PathRadar2):-4] for i in self.ArchRadar2
]
return
def Dates_datetime2str(self, DatesP, Date_Format=None):
'''
DESCRIPTION:
This function takes a Python date or datetime vector and return a
string data vector.
_______________________________________________________________________
INPUT:
+ DatesP: Python date or datetime vector.
+ Dates_Format: Format of the dates given, it must be given in
datetime string format like %Y/%m/%d %H%M.
_______________________________________________________________________
OUTPUT:
- Dates: Dates string vector.
'''
# ----------------
# Error managment
# ----------------
if isinstance(DatesP[0], datetime) == False and isinstance(
DatesP[0], date) == False:
Er = utl.ShowError(
'Dates_datetime2str', 'DatesUtil',
'Bad DatesP format given, not in date or datetime format')
raise TypeError
# -------------------------
# Date_Format Verification
# -------------------------
if Date_Format == None:
if isinstance(DatesP[0], datetime):
Date_Formats = self.DateTime_Formats
else:
Date_Formats = self.Date_Formats
else:
Date_Formats = [Date_Format]
# -------------------------
# Changing Dates
# -------------------------
Dates = np.array([i.strftime(Date_Formats[0]) for i in DatesP])
return Dates
def AddElim(self, Elim):
'''
DESCRIPTION:
This function loads the data of a station and compiles
it in a dictionary.
_____________________________________________________________
INPUT:
:param Elim: A Dict, dictionary with the values that would
be eliminated over and lower.
Ex: Elim = {'ElimOver':{'RSC':3000},
'ElimLow:{'TC':-1}}
'''
FlagKeyOver = True
FlagKeyLow = True
try:
ElimLabOver = list(Elim['ElimOver'])
except KeyError:
FlagKeyOver = False
except TypeError:
FlagKeyOver = False
try:
ElimLabLow = list(Elim['ElimLow'])
except KeyError:
FlagKeyLow = False
except TypeError:
FlagKeyLow = False
if not (FlagKeyOver) and not (FlagKeyLow):
r = utl.ShowError(
'AddElim', 'OpenWundergrounds',
'No elimination was added, review the Elim parameter')
raise KeyError
if FlagKeyOver:
for Lab in ElimLabOver:
self.ElimOver[Lab] = Elim['ElimOver'][Lab]
if FlagKeyLow:
for Lab in ElimLabLow:
self.ElimLow[Lab] = Elim['ElimLow'][Lab]
def EDIDEAM(File=None):
'''
DESCRIPTION:
With this function the information of an IDEAM type file can
be extracted.
_______________________________________________________________________
INPUT:
+ File: File that would be extracted including the path.
Default value is None because it works with Open_Data.
_______________________________________________________________________
OUTPUT:
'''
if File == None:
Er = utl.ShowError('EDIDEAM', 'EDSM', 'No file was added')
return None, None, None, None, None
# Match Variables
Station_Compile = re.compile('ESTACION')
Lat_Compile = re.compile('LATITUD')
Lon_Compile = re.compile('LONGITUD')
Elv_Compile = re.compile('ELEVACION')
Year_Compile = re.compile(' ANO ')
Var_Temp_Compile = re.compile('TEMPERATURA')
Var_Prec_Compile = re.compile('PRECIPITACION')
Var_BS_Compile = re.compile('BRILLO SOLAR')
Med_Temp_Compile = re.compile('VALORES MEDIOS')
Min_Temp_Compile = re.compile('VALORES MINIMOS')
Max_Temp_Compile = re.compile('VALORES MAXIMOS')
Start_Compile = re.compile('ENERO')
# Open the file
try:
f = open(File, 'r', encoding='utf-8')
Lines = np.array(f.readlines())
Sum1 = 6
Sum2 = 2
except TypeError:
f = open(File, 'r')
Lines1 = f.readlines()
Lines = np.array([i.encode('UTF-8') for i in Lines1])
Sum1 = 3
Sum2 = 1
if platform.system() == 'Windows':
Sum1 = 3
Sum2 = 1
# Variables para la estación
Station_Match = []
Row_Station = []
Station_Code = []
Station_Name = []
Lat_Match = []
Lats = []
Lon_Match = []
Lons = []
Elv_Match = []
Elvs = []
# Variables para el año
Year_Match = []
Row_Year = []
Years = []
# Estaciones con Temperatura
Var_Temp_Match = []
Row_Var_Temp = []
# Estaciones con Precipitación
Var_Prec_Match = []
Row_Var_Prec = []
# Estaciones con Brillo Solar
Var_BS_Match = []
Row_Var_BS = []
# Valores Medios
Med_Temp_Match = []
Row_Med_Temp = []
# Inicio de datos
Start_Match = []
Row_Start = []
keys = []
Stations_Code = []
Stations_Name = []
Value_Dict = dict()
DateP_Dict = dict()
Flags_Dict = dict()
x = 0
for irow, row in enumerate(Lines):
# Stations
if re.search(Station_Compile, row) != None:
Row_Station.append(irow)
Station_Match.append(re.search(Station_Compile, row))
Station_Code.append(row[Station_Match[-1].end() +
3:Station_Match[-1].end() + 3 + 8])
Station_Name.append(row[Station_Match[-1].end() + 3 + 8 + 1:-1])
# Latitude
if re.search(Lat_Compile, row) != None:
Lat_Match.append(re.search(Lat_Compile, row))
Lats.append(row[Lat_Match[-1].end() + 4:Lat_Match[-1].end() + 4 +
6])
# Longitude
if re.search(Lon_Compile, row) != None:
Lon_Match.append(re.search(Lon_Compile, row))
Lons.append(row[Lon_Match[-1].end() + 3:Lon_Match[-1].end() + 3 +
6])
# Elevation
if re.search(Elv_Compile, row) != None:
Elv_Match.append(re.search(Elv_Compile, row))
Elvs.append(row[Elv_Match[-1].end() + 2:Elv_Match[-1].end() + 2 +
4])
# Years
if re.search(Year_Compile, row) != None:
Row_Year.append(irow)
Year_Match.append(re.search(Year_Compile, row))
Years.append(row[Year_Match[-1].end() + 1:Year_Match[-1].end() +
1 + 4])
# Temperature
if re.search(Var_Temp_Compile, row) != None:
Row_Var_Temp.append(irow)
Var_Temp_Match.append('TEMPERATURA')
if re.search(Med_Temp_Compile, row) != None:
Row_Med_Temp.append(irow)
Med_Temp_Match.append('MEDIO')
elif re.search(Min_Temp_Compile, row) != None:
Med_Temp_Match.append('MINIMO')
elif re.search(Max_Temp_Compile, row) != None:
Med_Temp_Match.append('MAXIMO')
else:
Med_Temp_Match.append(0)
# Precipitation
if re.search(Var_Prec_Compile, row) != None:
Row_Var_Prec.append(irow)
Var_Temp_Match.append('PRECIPITACION')
Med_Temp_Match.append('TOTALES')
# Brillo Solar
if re.search(Var_BS_Compile, row) != None:
Row_Var_BS.append(irow)
Var_Temp_Match.append('BRILLO SOLAR')
Med_Temp_Match.append('TOTALES')
# Data
if re.search(Start_Compile, row) != None:
Row_Start.append(irow)
Start_Match.append(re.search(Start_Compile, row))
xRow_St = Row_Start[-1] + Sum1
xCol_St = Start_Match[-1].start()
if Row_Start[-1] <= 20:
DateP = []
Value = []
Flags = []
DateP, Value, Flags = LoopDataDaily_IDEAM(
DateP, Value, Flags, xRow_St, xCol_St, Years[-1], Lines,
Sum2)
elif (Station_Code[-1] == Station_Code[-2]) and (
Med_Temp_Match[-1]
== Med_Temp_Match[-2]) and (Var_Temp_Match[-1]
== Var_Temp_Match[-2]):
DateP, Value, Flags = LoopDataDaily_IDEAM(
DateP, Value, Flags, xRow_St, xCol_St, Years[-1], Lines,
Sum2)
else:
keys.append(Station_Code[-2] + '_' + Var_Temp_Match[-2] + '_' +
Med_Temp_Match[-2])
Stations_Name.append(Station_Name[-2])
Stations_Code.append(Station_Code[-2])
DateP_Dict[Station_Code[-2]+'_'+Var_Temp_Match[-2]+'_'+Med_Temp_Match[-2]]=\
DateP
Value_Dict[Station_Code[-2]+'_'+Var_Temp_Match[-2]+'_'+Med_Temp_Match[-2]]=\
Value
Flags_Dict[Station_Code[-2]+'_'+Var_Temp_Match[-2]+'_'+Med_Temp_Match[-2]]=\
Flags
DateP = []
Value = []
Flags = []
DateP, Value, Flags = LoopDataDaily_IDEAM(
DateP, Value, Flags, xRow_St, xCol_St, Years[-1], Lines,
Sum2)
if irow == len(Lines) - 1:
keys.append(Station_Code[-1] + '_' + Var_Temp_Match[-1] + '_' +
Med_Temp_Match[-1])
DateP_Dict[Station_Code[-1]+'_'+Var_Temp_Match[-1]+'_'+Med_Temp_Match[-1]]=\
DateP
Value_Dict[Station_Code[-1]+'_'+Var_Temp_Match[-1]+'_'+Med_Temp_Match[-1]]=\
Value
Flags_Dict[Station_Code[-1]+'_'+Var_Temp_Match[-1]+'_'+Med_Temp_Match[-1]]=\
Flags
# Station information
Stations_Code_Un, xU = np.unique(np.array(Station_Code), return_index=True)
Stations_Name_Un = np.array(Station_Name)[xU]
Lat_Un = np.array(Lats)[xU]
Lon_Un = np.array(Lons)[xU]
Elv_Un = np.array(Elvs)[xU]
Stations_Information = dict()
Stations_Information['CODE'] = Stations_Code_Un
Stations_Information['NAME'] = Stations_Name_Un
Stations_Information['LATITUDE'] = [i[:2] + '.' + i[2:] for i in Lat_Un]
Stations_Information['LONGITUDE'] = [i[:2] + '.' + i[2:] for i in Lon_Un]
Stations_Information['ELEVATION'] = Elv_Un
# Search for the Flags meaning
Flag_Meaning_Dict = dict()
keys = list(Flags_Dict)
for ikey, key in enumerate(keys):
if ikey == 0:
Flag_Un = np.unique(np.array(Flags_Dict[key]))
else:
Flag_Un = np.hstack(
(Flag_Un, np.unique(np.array(Flags_Dict[key]))))
Flag_Un2 = np.unique(Flag_Un)
# Removing the nan values
x = np.where(Flag_Un2 == 'nan')
Flag_Un2 = np.delete(Flag_Un2, x)
# Verify the flags meanining
Flag_Meaning = []
for flag in Flag_Un2:
Flag_Compile = re.compile(flag + ' :')
for irow, row in enumerate(Lines):
# Stations
if re.search(Flag_Compile, row) != None:
Flag_Match = re.search(Flag_Compile, row)
if Sum1 == 6:
Flag_Meaning.append(row[Flag_Match.start():-1])
else:
Flag_Meaning.append(row[Flag_Match.start():-3])
break
return DateP_Dict, Value_Dict, Flags_Dict, Flag_Meaning, Stations_Information
def EDExcel(File=None,
sheet=None,
colDates=(0, ),
colData=(1, ),
row_skip=1,
flagHeader=True,
row_end=None,
num_NaN=None):
'''
DESCRIPTION:
This function extracts time series data from a sheet in an Excel
file.
_______________________________________________________________________
INPUT:
+ File: File that needs to be open.
+ sheet: Index (number) or name (string) of the sheet.
+ colDates: tuple sequence with the columns where the dates or
string data is. Defaulted to (0). if None it extracts
all the data as the original format dictionary.
+ colData: tuple sequence with the columns where the data is.
Defaulted to (1). If None it extracts all the data
as a float data.
+ row_skip: begining row of the data. Defaulted to 1
+ flagHeader: flag to get the header of the information.
+ row_end: Ending row if nedded, defaulted to None.
+ num_NaN
_______________________________________________________________________
OUTPUT:
Dates and values are given in dictionaries.
'''
# ----------------
# Error Managment
# ----------------
# Verify values
if File == None:
Er = utl.ShowError('EDExcel', 'EDSM', 'No file was added')
return None, None, None
if sheet == None:
Er = utl.ShowError('EDExcel', 'EDSM', 'No sheet was added')
return None, None, None
if num_NaN == None:
flagnumNaN = False
else:
flagnumNaN = True
if isinstance(num_NaN, str):
Er = utl.ShowError('EDExcel', 'EDSM', 'num_NaN must be a number')
return None, None, None
num_NaN = float(num_NaN)
# ----------------
# Data extraction
# ----------------
flagName = False
flagcolDates = False
flagcolData = False
if isinstance(sheet, str):
flagName = True
if colDates == None:
flagcolDates = False
if colData == None:
flagcolData = False
# Open Excel File using xldr
B = xlrd.open_workbook(File)
# Se carga la página en donde se encuentra el información
if flagName:
S = B.sheet_by_name(sheet)
else:
S = B.sheet_by_index(sheet)
# Verify the number of columns and rows
ncol = S.ncols
nrow = S.nrows
if max(colDates) > ncol - 1 or max(colData) > ncol - 1:
Er = utl.ShowError('EDExcel', 'EDSM',
'column exceed dimension of the sheet')
if row_end != None:
if row_end > nrow - 1:
row_end = nrow - 1
else:
row_end = nrow - 1
# Header Exctraction
if flagHeader:
Header = S.row_values(row_skip - 1)
Header = list(np.array(Header)[list(colDates)]) + list(
np.array(Header)[list(colData)])
# Extracting time
Dates = dict()
Data = dict()
for iCDate, CDate in enumerate(colDates): # Revisar Fechas
Dates1 = S.col_values(CDate, start_rowx=row_skip, end_rowx=row_end)
try:
a = datetime(*xlrd.xldate_as_tuple(Dates1[0], B.datemode))
dif1 = xlrd.xldate_as_tuple(Dates1[1],
B.datemode)[3] - xlrd.xldate_as_tuple(
Dates1[1], B.datemode)[3]
dif2 = xlrd.xldate_as_tuple(Dates1[1],
B.datemode)[4] - xlrd.xldate_as_tuple(
Dates1[1], B.datemode)[4]
try:
dif3 = xlrd.xldate_as_tuple(
Dates1[1], B.datemode)[5] - xlrd.xldate_as_tuple(
Dates1[1], B.datemode)[5]
except IndexError:
dif3 = 0
if dif1 == 0 and dif2 == 0 and dif3 == 0:
Dates[CDate] = np.array([
datetime(*xlrd.xldate_as_tuple(i, B.datemode))
for i in Dates1
])
else:
Dates[CDate] = np.array([
datetime(*xlrd.xldate_as_tuple(i, B.datemode)).date()
for i in Dates1
])
except:
Data[CDate] = np.array(Dates1)
# Exctracting Data
for CData in colData: # Revisar Fechas
Data1 = S.col_values(CData, start_rowx=row_skip, end_rowx=row_end)
Data2 = []
# Verify data to become NaN
for dat in Data1:
try:
Data2.append(float(dat))
except ValueError:
Data2.append(np.nan)
Data2 = np.array(Data2)
if flagnumNaN:
x = np.where(Data2 == num_NaN)
Date2[x] = np.nan
Data[CData] = Data2
return Dates, Data, Header
def EDTXT(File,
deli=',',
colStr=(0, ),
colData=(1, ),
row_skip=1,
flagHeader=True,
rowH=0,
row_end=0,
str_NaN=None,
num_NaN=None,
dtypeData=float):
'''
DESCRIPTION:
This function extract data series from a plain text file or a csv
type file.
_______________________________________________________________________
INPUT:
:param File: A str, File that needs to be open with
extention.
:param deli: A str, Delimiter of the data. Defaulted to ','.
:param colStr: A tuple, tuple sequence with the columns where
the string data is. Defaulted to (0).
:param colData: A tuple, tuple sequence with the columns where
the floar data is. Defaulted to (1).
:param row_skip: An int, begining row. Defaulted to 0.
:param flagHeader: A boolean, flag to get the header of the
information.
:param rowH: A str, Header row.
:param row_end: A str, Ending row if nedded, defaulted to None.
:param str_NaN: A str, NaN string for data. Defaulted to None.
:param num_NaN: A str, NaN number for data. Defaulted to None.
:param dtypeData: A str, data type. Defaulted to float.
_______________________________________________________________________
OUTPUT:
Dates and values are given in dictionaries.
'''
# ----------------
# Error Managment
# ----------------
if not (isinstance(colStr, tuple)) and not (isinstance(
colStr, list)) and colStr != None:
Er = utl.ShowError('EDTXT', 'EMSD.Extract_Data',
'colStr not in tuple or list')
raise TypeError
elif not (isinstance(colStr, tuple)) and colStr != None:
colStr = tuple(colStr)
if not (isinstance(colData, tuple)) and not (isinstance(
colData, list)) and colData != None:
Er = utl.ShowError('EDTXT', 'EMSD.Extract_Data',
'colData not in tuple or list')
raise TypeError
elif not (isinstance(colData, tuple)) and colStr != None:
colData = tuple(colData)
# Verify values
if num_NaN == None:
flagnumNaN = False
else:
flagnumNaN = True
if isinstance(num_NaN, str) == False:
num_NaN = float(num_NaN)
if str_NaN == None:
flagstrNaN = False
else:
flagstrNaN = True
# -------------------
# Extracting Values
# -------------------
# Headers
if flagHeader:
if colStr == None and colData == None:
Headers = np.genfromtxt(File,
dtype=str,
skip_header=rowH,
delimiter=deli,
max_rows=1)
elif colStr != None and colData == None:
Headers = np.genfromtxt(File,
dtype=str,
usecols=colStr,
skip_header=rowH,
delimiter=deli,
max_rows=1)
elif colData != None and colStr == None:
Headers = np.genfromtxt(File,
dtype=str,
usecols=colData,
skip_header=rowH,
delimiter=deli,
max_rows=1)
else:
Headers = np.genfromtxt(File,
dtype=str,
usecols=colStr + colData,
skip_header=rowH,
delimiter=deli,
max_rows=1)
else:
rowH = 0
if colStr == None and colData == None:
Headers = np.genfromtxt(File,
dtype=str,
skip_header=rowH,
delimiter=deli,
max_rows=1)
elif colStr == None:
Headers = np.genfromtxt(File,
dtype=str,
usecols=colStr,
skip_header=rowH,
delimiter=deli,
max_rows=1)
elif colData == None:
Headers = np.genfromtxt(File,
dtype=str,
usecols=colData,
skip_header=rowH,
delimiter=deli,
max_rows=1)
else:
Headers = np.genfromtxt(File,
dtype=str,
usecols=colStr + colData,
skip_header=rowH,
delimiter=deli,
max_rows=1)
Headers = np.arange(0, len(Headers))
# R = {'Headers':Headers}
R = dict()
# String Data
if colStr != None:
DataStr = np.genfromtxt(File,
dtype=str,
usecols=colStr,
delimiter=deli,
skip_header=row_skip,
skip_footer=row_end,
unpack=True,
encoding='utf-8')
if flagstrNaN:
DataStr[DataStr == str_NaN] = 'nan'
if len(colStr) == 1:
R[Headers[0]] = DataStr
elif len(colStr) > 1:
for icol, col in enumerate(colStr):
R[Headers[icol]] = DataStr[icol]
if colData != None:
Data = np.genfromtxt(File,
dtype=dtypeData,
usecols=colData,
delimiter=deli,
skip_header=row_skip,
skip_footer=row_end,
unpack=True)
if flagnumNaN:
Data[Data == num_NaN] = np.nan
if len(colData) == 1:
if colStr == None:
R[Headers[icol]] = Data
else:
R[Headers[len(colStr) + 1]] = Data
elif len(colData) > 1:
for icol, col in enumerate(colData):
if colStr == None:
R[Headers[icol]] = Data[icol]
else:
R[Headers[len(colStr) + icol]] = Data[icol]
elif colData == None and colStr == None:
Data = np.genfromtxt(File,
dtype=dtypeData,
delimiter=deli,
skip_header=row_skip,
skip_footer=row_end,
unpack=True)
if flagnumNaN:
Data[Data == num_NaN] = np.nan
for icol, col in enumerate(Headers):
R[Headers[icol]] = Data[icol]
return R
def St_Document(Pathout='',
Name='Stations_Info',
St_Info_Dict=None,
Data_Flags=None):
'''
DESCRIPTION:
This function saves the station information in an Excel (.xlsx)
worksheet.
_______________________________________________________________________
INPUT:
+ Pathout: Saving directory.
+ Name: File Name.
+ St_Info_Dict: Dictionary with the information of the stations.
It must have the following information:
CODE: Station code.
NAME: Station name.
ELEVATION: Station Elevation.
LATITUDE: Station latitud.
LONGITUDE: Station Longitude.
+ Data_Flags: Possible flags that the data has defaulted to None.
_______________________________________________________________________
OUTPUT:
Return a document.
'''
if St_Info_Dict == None and self.St_Info == None:
Er = utl.ShowError('St_Document', 'EDSM', 'No station data added')
return
elif St_Info_Dict == None:
St_Info_Dict = self.St_Info
keys = ['CODE', 'NAME', 'ELEVATION', 'LATITUDE', 'LONGITUDE']
St_Info_Dict['LATITUDE'] = list(St_Info_Dict['LATITUDE'])
St_Info_Dict['LONGITUDE'] = list(St_Info_Dict['LONGITUDE'])
for i in range(len(St_Info_Dict['LATITUDE'])):
if St_Info_Dict['LATITUDE'][i][-1] == 'N':
St_Info_Dict['LATITUDE'][i] = float(
St_Info_Dict['LATITUDE'][i][:5])
elif St_Info_Dict['LATITUDE'][i][-1] == 'S':
St_Info_Dict['LATITUDE'][i] = float(
'-' + St_Info_Dict['LATITUDE'][i][:5])
for i in range(len(St_Info_Dict['LONGITUDE'])):
if St_Info_Dict['LONGITUDE'][i][-1] == 'E':
St_Info_Dict['LONGITUDE'][i] = float(
St_Info_Dict['LONGITUDE'][i][:5])
elif St_Info_Dict['LONGITUDE'][i][-1] == 'W':
St_Info_Dict['LONGITUDE'][i] = float(
'-' + St_Info_Dict['LONGITUDE'][i][:5])
Nameout = Pathout + Name + '.xlsx'
W = xlsxwl.Workbook(Nameout)
# Stations Sheet
WS = W.add_worksheet('STATIONS')
# Cell formats
Title = W.add_format({'bold': True,'align': 'center','valign': 'vcenter'\
,'font_name':'Arial','font_size':11,'top':1,'bottom':1,'right':1\
,'left':1})
Data_Format = W.add_format({'bold': False,'align': 'left','valign': 'vcenter'\
,'font_name':'Arial','font_size':11,'top':1,'bottom':1,'right':1\
,'left':1})
# Column Formats
WS.set_column(1, 3, 20.0)
WS.set_column(1, 3, 20.0)
WS.set_column(1, 4, 15.0)
WS.set_column(1, 5, 15.0)
# Titles
WS.write(1, 1, 'CODE', Title)
WS.write(1, 2, 'NAME', Title)
WS.write(1, 3, 'ELEVATION', Title)
WS.write(1, 4, 'LATITUDE (N)', Title)
WS.write(1, 5, 'LONGITUDE (E)', Title)
Col = 1
for key in keys:
Row = 2
for dat in St_Info_Dict[key]:
WS.write(Row, Col, dat, Data_Format)
Row += 1
Col += 1
if Data_Flags == None and self.FlagM == None:
W.close()
return
elif Data_Flags == None:
Data_Flags = self.FlagM
WS = W.add_worksheet('FLAGS')
WS.write(1, 1, 'FLAGS', Title)
WS.set_column(1, 1, 20.0)
Row = 2
for flag in Data_Flags:
WS.write(Row, 1, flag, Data_Format)
Row += 1
W.close()
return
def __init__(self, PathData, Stations=None):
'''
DESCRIPTION:
Class Constructor.
'''
# -------------------------
# Error Managment
# -------------------------
if not (isinstance(Stations, list)) and not (isinstance(
Stations, str)) and Stations != None:
r = utl.ShowError('OpenWundergrounds', '__init__',
'Erroneus type for Station information')
raise TypeError
# -------------------------
# Parameters
# -------------------------
self.PathData = PathData
self.deli = ',' # Delimeter
# Important Variables
# String Variables
self.LabelsStr = np.array(['Time'])
self.LabelsData = np.array([
'TemperatureC', 'DewpointC', 'PressurehPa', 'WindSpeedKMH',
'WindDirectionDegrees', 'HourlyPrecipMM',
'SolarRadiationWatts/m^2', 'Humidity'
])
self.LabDataOper = {
'TemperatureC': 'mean',
'DewpointC': 'mean',
'PressurehPa': 'mean',
'WindSpeedKMH': 'mean',
'WindDirectionDegrees': 'mean',
'HourlyPrecipMM': 'sum',
'SolarRadiationWatts/m^2': 'mean',
'Humidity': 'mean'
}
self.LabDataOper1 = {
'TemperatureC': 'mean',
'DewpointC': 'mean',
'PressurehPa': 'mean',
'WindSpeedKMH': 'mean',
'WindDirectionDegrees': 'mean',
'HourlyPrecipMM': 'sum',
'SolarRadiationWatts/m^2': 'mean',
'Humidity': 'mean'
}
self.LabDataOper2 = {
'TemperatureC': 'mean',
'DewpointC': 'mean',
'PressurehPa': 'mean',
'WindSpeedKMH': 'mean',
'WindDirectionDegrees': 'mean',
'HourlyPrecipMM': 'sum',
'SolarRadiationWatts/m^2': 'sum',
'Humidity': 'mean'
}
self.LabDataOper3 = {
'TemperatureC': 'mean',
'DewpointC': 'mean',
'PressurehPa': 'mean',
'WindSpeedKMH': 'mean',
'WindDirectionDegrees': 'mean',
'HourlyPrecipMM': 'sum',
'SolarRadiationWatts/m^2': 'mean',
'Humidity': 'mean'
}
self.LabDataSave = {
'TemperatureC': 'TC',
'DewpointC': 'Td',
'PressurehPa': 'PresC',
'WindSpeedKMH': 'WSC',
'WindDirectionDegrees': 'WDC',
'HourlyPrecipMM': 'PrecC',
'SolarRadiationWatts/m^2': 'RSC',
'Humidity': 'HRC'
}
self.ElimOver = {
'TC': None,
'Td': None,
'PresC': None,
'WSC': None,
'WDC': None,
'PrecC': None,
'RSC': None,
'HRC': None
}
self.ElimLow = {
'TC': None,
'Td': None,
'PresC': None,
'WSC': None,
'WDC': None,
'PrecC': None,
'RSC': None,
'HRC': None
}
self.LabelsWithUnits = {
'TC': 'Temperatura [°C]',
'Td': 'Punto de Rocio [°C]',
'PresC': 'Presión [hPa]',
'WSC': 'Vel. Viento [m/s]',
'WDC': 'Dirección del Viento [Grados]',
'PrecC': 'Precipitación [mm]',
'RSC': r'Radiación Solar [W/m$^2$]',
'HRC': 'Hum. Rel. [%]'
}
self.LabelsNoUnits = {
'TC': 'Temperatura',
'Td': 'Punto de Rocio',
'PresC': 'Presión',
'WSC': 'Vel. Viento',
'WDC': 'Dirección del Viento',
'PrecC': 'Precipitación',
'RSC': r'Radiación Solar',
'HRC': 'Hum. Rel.'
}
self.LabelsColors = {
'TC': 'r',
'Td': 'r',
'PresC': 'k',
'WSC': 'k',
'WDC': 'k',
'PrecC': 'b',
'RSC': 'y',
'HRC': 'g'
}
# -------------------------
# Get Stations
# -------------------------
if Stations == None:
Stations = utl.GetFolders(PathData)
elif isinstance(Stations, str):
Stations = [Stations]
# Stations
self.Stations = Stations
self.Arch = dict()
for St in Stations:
self.Arch[St] = gl.glob(PathData + St + '/*.txt')
if len(self.Arch[St]) == 0:
print('In Station', St, 'No data was found, review station')
return
def CiclD(Var, Years=None, Dates=None, DTH=24, flagZeros=False):
'''
DESCRIPTION:
This function calculates the diurnal cycle of a variable from the
hourly data of a time series, it would obtain the monthly diurnal
cycle from all the different months and the total.
_____________________________________________________________________
INPUT:
:param Var: A list or ndarray, Variable that need to be
treated.
:param Years: A list or ndarray, Vector with the begining year and the ending year.
:param Dates: A list or ndarray, Vector with the dates in string
format yyyy/mm/dd HHMM.
:param DTH: A int, Determine the number of data that
takes to complete a day it is
defaulted to 24.
:param flagZeros: A boolean, flag to know if the zeros are taking
into account.
_____________________________________________________________________
OUTPUT:
:return Resutls: A dict, The dictionary has:
:return MonthsM: A dict, Dictionary with the Monthly data.
:return MonthsMM: A dict, Dictionary with the results
per month.
:return MonthsME: A dict, Dictionary with the mean errors
per month.
:return CiDT: A ndarray, Mean complete diurnal cycle.
:return ErrT: A ndarray, Mean Errors.
'''
# Errors
if Years == None and Dates == None:
Er = utl.ShowError('CicloD', 'An_Hydro',
'No dates nor years were added')
return
elif Dates == None:
FlagYears = True
else:
FlagYears = False
# Variables
# Months
MonthsM = dict() # Data
MonthsMM = dict() # Mean Data
MonthsMMM = dict()
MonthsMD = dict() # Standard deviation
MonthsME = dict() # Median Error
TriM = dict() # Data trimestrales
TriMM = dict() # Mean trimestral
TriMD = dict() # Standard Deviation trimestral
TriME = dict() # Median Error trimestral
# Dates
if FlagYears:
Yi = int(Years[0])
Yf = int(Years[1])
dt = timedelta(0, 24 / DTH * 3600)
DateI = datetime(Years[0], 1, 1, 0, 0)
DateI = datetime(Years[1], 12, 31, 23, 59)
Date = DUtil.Dates_Comp(DateI, DateE, dtm=dt)
elif isinstance(Dates[0], str):
Date = DUtil.Dates_str2datetime(Dates)
else:
Date = Dates
Months = np.array([i.month for i in Date])
if flagZeros:
q = np.where(Var == 0)
Var[q] = np.nan
# Months
MaxV = []
MinV = []
for i in range(1, 13):
x = np.where(Months == i)
MonthsM[i] = np.reshape(np.copy(Var)[x], (-1, DTH))
MonthsMM[i], MonthsMD[i], MonthsME[i] = MeanError(MonthsM[i], axis=0)
MaxV.append(np.nanmax(MonthsMM[i]) + np.nanmax(MonthsME[i] * 1.2))
MinV.append(
np.nanmin(MonthsMM[i]) - np.abs(np.nanmin(MonthsME[i] * 1.2)))
Trimes = {1: [12, 1, 2], 2: [3, 4, 5], 3: [6, 7, 8], 4: [9, 10, 11]}
for i in range(1, 5):
TriM[i] = np.reshape(
np.copy(Var)[(Months == Trimes[i][0]) | (Months == Trimes[i][1]) |
(Months == Trimes[i][2])], (-1, DTH))
TriMM[i], TriMD[i], TriME[i] = MeanError(TriM[i], axis=0)
# Reshaped variable
VarM = np.reshape(np.copy(Var), (-1, DTH))
CiDT, DesT, ErrT = MeanError(VarM, axis=0)
Results = {
'MonthsM': MonthsM,
'MonthsMM': MonthsMM,
'MonthsME': MonthsME,
'CiDT': CiDT,
'ErrT': ErrT,
'DesT': DesT,
'TriM': TriM,
'TriMM': TriMM,
'TriMD': TriMD,
'TriME': TriME
}
return Results
def EDnetCDFFile(File, VarDict=None, VarRangeDict=None):
'''
DESCRIPTION:
With this function the information of an IDEAM type file can
be extracted.
_______________________________________________________________________
INPUT:
+ File: File that would be extracted including the path.
+ VarDict: List of variables that would be extracted from the
netCDF file. Defaulted to None.
+ VarRangeDict: Range of data that would be extracted per
variable. It is defaulted to None if all the
Range wants to be extracted.
It must be a list with two values for each variable.
_______________________________________________________________________
OUTPUT:
- Data: Extracted Data Dictionary.
'''
# Importing netCDF libary
try:
import netCDF4 as nc
except:
Er = utl.ShowError(
'EDNCFile', 'EDSM',
'netCDF4 not installed, please install the library to continue')
raise Er
# Open File
dataset = nc.Dataset(File)
if VarDict == None:
Data = dataset.variables
else:
Data = dict()
for iVar, Var in enumerate(VarDict):
try:
P = dataset.variables[Var]
except KeyError:
Er = utl.ShowError('EDNCFile', 'EDSM',
'Key %s not in the nc file.' % Var)
raise Er
if VarRangeDict == None:
if Var == 'time':
Data[Var] = nc.num2date(dataset.variables[Var][:],
dataset.variables[Var].units,
dataset.variables[Var].calendar)
else:
Data[Var] = dataset.variables[Var][:]
else:
a = dataset.variables[Var] # Variable
dimensions = a.dimensions # Keys of dimensions
LenD = len(dimensions) # Number of dimensions
totalshape = a.shape # Shape of the matrix
Range = dict()
for iVarR, VarR in enumerate(dimensions):
try:
Range[VarR] = VarRangeDict[VarR]
except:
Range[VarR] = [0, dataset.variables[VarR].shape[0]]
if LenD == 1:
if Var == 'time':
Data[Var] = nc.num2date(
dataset.variables[Var][:],
dataset.variables[Var].units,
dataset.variables[Var].calendar)[slice(
Range[dimensions[0]][0],
Range[dimensions[0]][1])]
else:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0], Range[dimensions[0]][1])]
elif LenD == 2:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0],
Range[dimensions[0]][1])][slice(
Range[dimensions[1]][0], Range[dimensions[1]][1])]
elif LenD == 3:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0],
Range[dimensions[0]][1])][slice(
Range[dimensions[1]][0],
Range[dimensions[1]][1])][slice(
Range[dimensions[2]][0],
Range[dimensions[2]][1])]
elif LenD == 4:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0],
Range[dimensions[0]][1])][slice(
Range[dimensions[1]][0],
Range[dimensions[1]][1])][slice(
Range[dimensions[2]][0],
Range[dimensions[2]][1])][slice(
Range[dimensions[3]][0],
Range[dimensions[3]][1])]
elif LenD == 5:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0],
Range[dimensions[0]][1])][slice(
Range[dimensions[1]][0],
Range[dimensions[1]][1])][slice(
Range[dimensions[2]][0],
Range[dimensions[2]][1])][slice(
Range[dimensions[3]][0],
Range[dimensions[3]][1])][slice(
Range[dimensions[4]][0],
Range[dimensions[4]][1])]
dataset.close()
return Data
def Ca_E(FechaC,V1C,dt=24,escala=1,op='mean',flagMa=False,flagDF=False,flagNaN=True):
'''
DESCRIPTION:
Con esta función se pretende cambiar de escala temporal los datos,
agregándolos a diferentes escalas temporales, se deben insertar series
completas de tiempo.
Los datos faltantes deben estar como NaN.
_______________________________________________________________________
INPUT:
+ FechaC: Fecha de los datos organizada como 'año/mes/dia - HHMM'
los '/' pueden ser substituidos por cualquier caracter.
Debe ser un vector string y debe tener años enteros.
+ V1C: Variable que se desea cambiar de escala temporal.
+ dt: Delta de tiempo para realizar la agregación, depende de
la naturaleza de los datos.
Si se necesitan datos mensuales, el valor del dt debe ser 1.
+ escala: Escala a la cual se quieren pasar los datos:
-1: de minutal.
0: horario.
1: a diario.
2: a mensual, es necesario llevarlo primero a escala diaria.
+ op: Es la operación que se va a llevar a cabo para por ahora solo responde a:
'mean': para obtener el promedio.
'sum': para obtener la suma.
+ flagMa: Para ver si se quieren los valores máximos y mínimos.
True: Para obtenerlos.
False: Para no calcularos.
+ flagDF: Para ver si se quieren los datos faltantes por mes, solo funciona
en los datos que se dan diarios.
True: Para calcularlos.
False: Para no calcularos.
+ flagNaN: Flag to know if the user wants to include the calculations with low data.
_______________________________________________________________________
OUTPUT:
- FechaEs: Nuevas fechas escaladas.
- FechaNN: Nuevas fechas escaladas como vector fechas.
- VE: Variable escalada.
- VEMax: Vector de máximos.
- VEMin: Vector de mínimos.
'''
# Se desactivan los warnings en este codigo para que corra más rápido, los
# warnings que se generaban eran por tener realizar promedios de datos NaN
# no porque el código tenga un problema en los cálculos.
warnings.filterwarnings('ignore')
if escala > 2:
utl.ShowError('EMSD','Ca_E','Todavía no se han programado estas escalas')
# -------------------------------------------
# Inicialización de variables
# -------------------------------------------
# Se inicializan las variables que se utilizarán
FechaNN = ["" for k in range(1)]
FechaEs = ["" for k in range(1)]
VE = []
VEMax = []
VEMin = []
NF = [] # Porcentaje de datos faltantes
NNF = [] # Porcentaje de datos no faltantes
rr = 0
Oper = {'sum':np.nansum,'mean':np.nanmean}
# -------------------------------------------
# Vector de fechas
# -------------------------------------------
# Se toman los años
yeari = int(FechaC[0][0:4]) # Año inicial
yearf = int(FechaC[len(FechaC)-1][0:4]) # Año final
Sep = FechaC[0][4] # Separador de la Fecha
if isinstance(FechaC[0],str):
DatesO = DUtil.Dates_str2datetime(FechaC)
else:
DatesO = FechaC
# Los años se toman para generar el output de FechasEs
if escala == -1:
DateI = datetime(DatesO[0].year,1,1,0,0)
DateE = datetime(DatesO[-1].year,12,31,23,59)
dtm = timedelta(0,dt*60)
FechaNN = DUtil.Dates_Comp(DateI,DateE,dtm=dtm)
FechaEs = DUtil.Dates_datetime2str(FechaNN)
elif escala == 0:
DateI = datetime(DatesO[0].year,1,1,0,0)
DateE = datetime(DatesO[-1].year,12,31,23,59)
dtm = timedelta(0,60*60)
FechaNN = DUtil.Dates_Comp(DateI,DateE,dtm=dtm)
FechaEs = DUtil.Dates_datetime2str(FechaNN)
elif escala == 1: # Para datos horarios o diarios
for result in perdelta(date(int(yeari), 1, 1), date(int(yearf)+1, 1, 1), timedelta(days=1)):
FR = result.strftime('%Y'+Sep+'%m'+Sep+'%d') # Fecha
if escala == 0:
for i in range(0,24):
if rr == 0:
FechaNN[0] = result
if i < 10:
FechaEs[rr] = FR + '-0' +str(i)+'00'
else:
FechaEs[rr] = FR + '-' +str(i)+'00'
else:
FechaNN.append(result)
if i < 10:
FechaEs.append(FR + '-0' +str(i)+'00')
else:
FechaEs.append(FR + '-' +str(i)+'00')
rr += 1 # Se suman las filas
elif escala == 1:
if rr == 0:
FechaNN[0] = result
FechaEs[rr] = FR
else:
FechaNN.append(result)
FechaEs.append(FR)
rr += 1
if escala == 2:
x = 0
for i in range(int(yeari),int(yearf)+1):
for j in range(1,13):
if i == int(yeari) and j == 1:
FechaNN[0] = date(i,j,1)
FechaEs[0] = FechaNN[0].strftime('%Y'+Sep+'%m')
else:
FechaNN.append(date(i,j,1))
FechaEs.append(FechaNN[x].strftime('%Y'+Sep+'%m'))
x += 1
# -------------------------------------------
# Cálculo del escalamiento
# -------------------------------------------
dtt = 0 # Contador de la diferencia
if op == 'mean':
if escala == 0 or escala == -1 or escala == 1:
# Ciclo para realizar el agregamiento de los datos
for i in range(0,len(V1C),dt):
dtt = dtt + dt # Se aumenta el número de filas en el contador
q = np.isnan(V1C[i:dtt])
qq = sum(q)
qYes = sum(~np.isnan(V1C[i:dtt]))
if (qq > dt*0.30 and flagNaN) or qYes == 0:
VE.append(np.nan)
if flagMa == True:
VEMax.append(np.nan)
VEMin.append(np.nan)
else:
try:
VE.append(float(np.nanmean(V1C[i:dtt])))
except ValueError:
VE.append(np.nan)
if flagMa == True:
try:
VEMax.append(float(np.nanmax(V1C[i:dtt])))
except ValueError:
VEMax.append(np.nan)
try:
VEMin.append(float(np.nanmin(V1C[i:dtt])))
except ValueError:
VEMin.append(np.nan)
elif op == 'sum':
if escala == 0 or escala == -1 or escala == 1:
# Ciclo para realizar el agregamiento de los datos
for i in range(0,len(V1C),dt):
dtt = dtt + dt # Se aumenta el número de filas en el contador
q = np.isnan(V1C[i:dtt])
qq = sum(q)
qYes = sum(~np.isnan(V1C[i:dtt]))
if (qq > dt*0.30 and flagNaN) or qYes == 0:
VE.append(np.nan)
if flagMa == True:
VEMax.append(np.nan)
VEMin.append(np.nan)
else:
try:
VE.append(float(np.nansum(V1C[i:dtt])))
except ValueError:
VE.append(np.nan)
if flagMa == True:
try:
VEMax.append(float(np.nanmax(V1C[i:dtt])))
except ValueError:
VEMax.append(np.nan)
try:
VEMin.append(float(np.nanmin(V1C[i:dtt])))
except ValueError:
VEMin.append(np.nan)
if escala == 2:
YearMonthData = np.array([str(i.year)+'/'+str(i.month) for i in DatesO])
YearMonth = np.array([str(date(i,j,1).year)+'/'+str(date(i,j,1).month) for i in range(int(yeari),int(yearf)+1) for j in range(1,13)])
VE = np.empty(YearMonth.shape)*np.nan
VEMax = np.empty(YearMonth.shape)*np.nan
VEMin = np.empty(YearMonth.shape)*np.nan
NF = np.empty(YearMonth.shape)*np.nan
NNF = np.empty(YearMonth.shape)*np.nan
for iYM, YM in enumerate(YearMonth):
x = np.where(YearMonthData == YM)[0]
if len(x) != 0:
q = sum(~np.isnan(V1C[x]))
NF[iYM] = (q/len(x))
NNF[iYM] = (1-NF[-1])
if q >= round(len(x)*0.7,0) and flagNaN:
VE[iYM] = Oper[op](V1C[x])
VEMax[iYM] = np.nanmax(V1C[x])
VEMin[iYM] = np.nanmin(V1C[x])
# -------------------------------------------
# Se dan los resultados
# -------------------------------------------
if flagMa == True:
if flagDF:
return np.array(FechaEs), np.array(FechaNN), np.array(VE), np.array(VEMax), np.array(VEMin), np.array(NF),np.array(NNF)
else:
return np.array(FechaEs), np.array(FechaNN), np.array(VE), np.array(VEMax), np.array(VEMin)
elif flagMa == False:
if flagDF:
return np.array(FechaEs), np.array(FechaNN), np.array(VE),np.array(NF),np.array(NNF)
else:
return np.array(FechaEs), np.array(FechaNN), np.array(VE)
def CiclA(VMes, Years, flagA=False, oper='mean'):
'''
DESCRIPTION:
This function calculates the annual cycle of a variable, also
calculates the annual series of requiered.
Additionally, this function can makes graphs of the annual cycle
and the annual series if asked.
_______________________________________________________________________
INPUT:
:param VMes: A ndarray, Variable with the monthly data.
:param Years: A list or ndarray, Vector with the initial and
final year.
:param flagA: A boolean, flag to know if the annual series
is requiered.
:param oper: A ndarray, Operation for the annual data.
_______________________________________________________________________
OUTPUT:
'''
# --------------------
# Error Managment
# --------------------
if len(Years) > 2:
return utl.ShowError(
'CiclA', 'Hydro_Analysis',
'Years index vector larger than 2, review vector')
# --------------------
# Years Managment
# --------------------
Yi = int(Years[0])
Yf = int(Years[1])
VarM = np.reshape(VMes, (-1, 12))
# --------------------
# Annual Cycle
# --------------------
# Verify NaN data from the cycle
MesM = np.empty(12)
VarMNT = []
for i in range(12):
q = sum(~np.isnan(VarM[:, i]))
VarMNT.append(sum(~np.isnan(VarM[:, i])))
if q < round(len(VarM[:, i]) * 0.70, 0):
MesM[i] = np.nan
else:
MesM[i] = np.nanmean(VarM[:, i]) # Multianual Mean
MesD = np.nanstd(VarM, axis=0) # annual strandard deviation.
MesE = np.array([k / np.sqrt(VarMNT[ii])
for ii, k in enumerate(MesD)]) # annual Error
# --------------------
# Annual Series
# --------------------
if flagA:
# Determine operation
Operation = DM.Oper_Det(oper)
# ----------------
# Error managment
# ----------------
if Operation == -1:
return -1
# Calculations
AnM = np.empty(VarM.shape[0])
AnMNT = []
for i in range(VarM.shape[0]):
q = sum(~np.isnan(VarM[i, :]))
if q <= len(VarM[i, :]) * 0.70:
AnM[i] = np.nan
AnMNT.append(np.nan)
else:
AnM[i] = Operation(VarM[i, :])
AnMNT.append(q)
AnD = np.nanstd(VarM, axis=1) # Annual deviation
AnE = np.array([k / np.sqrt(AnMNT[ii])
for ii, k in enumerate(AnD)]) # Annual Error
# Return values
results = dict()
if flagA:
results['MesM'] = MesM
results['MesD'] = MesD
results['MesE'] = MesE
results['AnM'] = AnM
results['AnD'] = AnD
results['AnE'] = AnE
return results
else:
results['MesM'] = MesM
results['MesD'] = MesD
results['MesE'] = MesE
return results
def EDnetCDFFile(File,
VarDict=None,
VarRangeDict=None,
time='time',
DateI=None):
'''
DESCRIPTION:
With this function the information of an netCDF type file can
be extracted.
_______________________________________________________________________
INPUT:
:param File: A str, File that would be extracted including
the path.
:param VarDict: A dict, List of variables that would be
extracted from the netCDF file. Defaulted
to None.
:param VarRangeDict: A dict, Range of data that would be extracted
per variable. It is defaulted to None if all
the Range wants to be extracted.
It must be a list with two values for each
variable.
:param time: A str, key string of the time data in the
NetCDF file.
:param DateI: A date or datetime, object with the initial
date.
_______________________________________________________________________
OUTPUT:
:return Data: A dict, Extracted Data Dictionary.
'''
# Importing netCDF libary
try:
import netCDF4 as nc
except ImportError:
utl.ShowError(
'EDNCFile', 'EDSM',
'netCDF4 not installed, please install the library to continue')
# Open File
dataset = nc.Dataset(File)
if VarDict == None:
Data = dataset.variables
else:
Data = dict()
for iVar, Var in enumerate(VarDict):
try:
P = dataset.variables[Var]
except KeyError:
utl.ShowError('EDNCFile', 'EDSM',
'Key %s not in the nc file.' % Var)
if VarRangeDict == None:
if Var == time:
if dataset.variables[Var].calendar == '360':
FlagMonths = False
MStr = re.compile('months')
MMatch = re.search(MStr, dataset.variables[Var].units)
Data[Var] = []
if not (MMatch is None):
if not (DateI is None):
for An in range(
DateI.year, DateI.year + int(
len(dataset.variables[Var][:]) /
12)):
for M in range(1, 13):
if M < DateI.month and A == DateI.year:
continue
else:
Data[Var].append(date(An, M, 1))
Data[Var] = np.array(Data[Var])
else:
Data[Var] = nc.num2date(
dataset.variables[Var][:],
dataset.variables[Var].units,
dataset.variables[Var].calendar)
else:
Data[Var] = dataset.variables[Var][:]
else:
a = dataset.variables[Var] # Variable
dimensions = a.dimensions # Keys of dimensions
LenD = len(dimensions) # Number of dimensions
totalshape = a.shape # Shape of the matrix
Range = dict()
for iVarR, VarR in enumerate(dimensions):
try:
Range[VarR] = VarRangeDict[VarR]
except:
Range[VarR] = [0, dataset.variables[VarR].shape[0]]
if LenD == 1:
if Var == time:
Data[Var] = nc.num2date(
dataset.variables[Var][:],
dataset.variables[Var].units,
dataset.variables[Var].calendar)[slice(
Range[dimensions[0]][0],
Range[dimensions[0]][1])]
else:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0], Range[dimensions[0]][1])]
elif LenD == 2:
Data[Var] = dataset.variables[Var][slice(
Range[dimensions[0]][0], Range[dimensions[0]][1]), :]
Data[Var] = dataset.variables[
Var][:,
slice(Range[dimensions[1]][0], Range[
dimensions[1]][1])]
elif LenD == 3:
Data[Var] = dataset.variables[Var][
slice(Range[dimensions[0]][0], Range[dimensions[0]][1]
), :, :]
Data[Var] = dataset.variables[
Var][:,
slice(Range[dimensions[1]][0], Range[
dimensions[1]][1]), :]
Data[Var] = dataset.variables[
Var][:, :,
slice(Range[dimensions[2]][0], Range[
dimensions[2]][1])]
elif LenD == 4:
Data[Var] = dataset.variables[Var][
slice(Range[dimensions[0]][0], Range[dimensions[0]][1]
), :, :, :]
Data[Var] = dataset.variables[
Var][:,
slice(Range[dimensions[1]][0], Range[
dimensions[1]][1]), :, :]
Data[Var] = dataset.variables[
Var][:, :,
slice(Range[dimensions[2]][0], Range[
dimensions[2]][1]), :]
Data[Var] = dataset.variables[
Var][:, :, :,
slice(Range[dimensions[3]][0], Range[
dimensions[3]][1])]
elif LenD == 5:
Data[Var] = dataset.variables[Var][
slice(Range[dimensions[0]][0], Range[dimensions[0]][1]
), :, :, :, :]
Data[Var] = dataset.variables[
Var][:,
slice(Range[dimensions[1]][0], Range[
dimensions[1]][1]), :, :, :]
Data[Var] = dataset.variables[
Var][:, :,
slice(Range[dimensions[2]][0], Range[
dimensions[2]][1]), :, :]
Data[Var] = dataset.variables[
Var][:, :, :,
slice(Range[dimensions[3]][0], Range[
dimensions[3]][1]), :]
Data[Var] = dataset.variables[
Var][:, :, :, :,
slice(Range[dimensions[4]][0], Range[
dimensions[4]][1])]
dataset.close()
return Data
def PrecCount(Prec,DatesEv,dt=1,M=60):
'''
DESCRIPTION:
This functions calculates the duration of precipitation events
from composites.
_________________________________________________________________________
INPUT:
:param PrecC: A ndarray, Array with composite of precipitation.
:param DatesEv: A ndarray, Array with all the events dates, format
yyyy/mm/dd-HHMM or datetime.
:param dt: An int, Time delta in minutes.
:param M: An int, Place where the maximum of precipitation
is presented.
_________________________________________________________________________
OUTPUT:
:return DurPrec: A ndarray, Precipitation duration in hours.
:return TotalPrec: A ndarray, Total of precipitation in that time.
:return IntPrec: A ndarray, Event Intensity.
:return MaxPrec: A ndarray, Maximum of precipitation during the event.
:return DatesEvst: A ndarray, Date where the event begins.
:return DatesEvend: A ndarray, Date where the event ends.
'''
# --------------------------------------
# Error managment
# --------------------------------------
# --------------------------------------
# Dates
# --------------------------------------
# Manage Data Size
if len(DatesEv.shape) == 1:
if not(isinstance(DatesEv[0],str)):
E = utl.ShowError('PrecCount','MeteoFunctions','Not dates given, review format')
raise E
EvN = 1 # Events number
else:
if not(isinstance(DatesEv[0][0],str)):
E = utl.ShowError('PrecCount','MeteoFunctions','Not dates given, review format')
raise E
EvN = len(DatesEv) # Events number
# Variables for benining and end of the event
DatesEvst_Aft = []
DatesEvend_Aft = []
for i in range(EvN):
if isinstance(M,list):
MP = M[i]
else:
MP = M
x = [MP]
# Minimum of precipitation
if dt == 1:
MinPrec = 0.001
else:
MinPrec = 0.10
# Precipitation beginning
if EvN == 1:
xm = np.where(Prec[:MP]<=MinPrec)[0]
else:
xm = np.where(Prec[i,:MP]<=MinPrec)[0]
k = 1
a = len(xm)-1
I = 10
while k == 1:
if dt == 1:
if a == -1:
xmm = 0
k = 2
break
while a-I < 0:
I -= 1
if xm[a] == xm[a-I]+I:
xmm = xm[a]
k = 2
else:
a = a-1
if a == 0:
xmm = xm[0]
k = 2
elif dt == 5:
if a == -1:
xmm = 0
k = 2
break
if xm[a] == xm[a-1]+1:
xmm = xm[a]
k = 2
else:
a = a-1
if a == 0:
xmm = xm[0]
k = 2
# Precipitation ending
if EvN == 1:
xM = np.where(Prec[x[0]+1:]<=MinPrec)[0]+x[0]+1
else:
xM = np.where(Prec[i,x[0]+1:]<=MinPrec)[0]+x[0]+1
k = 1
a = 0
while k == 1:
aa = len(xM)
if aa == 1 or aa == 0:
if EvN == 1:
xMM = len(Prec)-1
else:
xMM = len(Prec[i,:])-1
k = 2
break
if dt == 1:
# print('a',a)
try:
if xM[a] == xM[a+10]-10:
xMM = xM[a]
k = 2
else:
a = a+1
if a == len(xM)-1:
xMM = xM[len(xM)-1]
k = 2
except:
try:
if xM[a] == xM[a+5]-5:
xMM = xM[a]
k = 2
else:
a = a+1
if a == len(xM)-1:
xMM = xM[len(xM)-1]
k = 2
except:
xMM = xM[a]
k = 2
elif dt == 5:
if xM[a] == xM[a+1]-1:
xMM = xM[a]
k = 2
else:
a = a+1
if a == len(xM)-1:
xMM = xM[len(xM)-1]
k = 2
else:
xMM = xM[a]
k = 2
if EvN == 1:
DatesEvst_Aft.append(DatesEv[xmm])
DatesEvend_Aft.append(DatesEv[xMM])
else:
DatesEvst_Aft.append(DatesEv[i][xmm])
DatesEvend_Aft.append(DatesEv[i][xMM])
DatesEvst = DUtil.Dates_str2datetime(DatesEvst_Aft,Date_Format=None)
DatesEvend = DUtil.Dates_str2datetime(DatesEvend_Aft,Date_Format=None)
DatesEvst_Aft = np.array(DatesEvst_Aft)
DatesEvend_Aft = np.array(DatesEvend_Aft)
# ---------------
# Calculations
# ---------------
# Variables
DurPrec = []
TotalPrec = []
IntPrec = []
IntPrecMax = []
MaxPrec = []
Pindex = []
TasaPrec = []
DatesMax = []
if EvN == 1:
# Verify event data
q = sum(~np.isnan(Prec))
if q <= len(DatesEv)*.90:
DurPrec.append(np.nan)
TotalPrec.append(np.nan)
IntPrec.append(np.nan)
IntPrecMax.append(np.nan)
MaxPrec.append(np.nan)
Pindex.append(np.nan)
TasaPrec.append(np.nan)
DatesMax.append(np.nan)
else:
# ------------------------
# Rainfall duration
# ------------------------
Dxi = np.where(DatesEv == DatesEvst_Aft)[0]
Dxf = np.where(DatesEv == DatesEvend_Aft)[0]
DurPrec.append((Dxf[0]-Dxi[0]+1)*dt/60) # Duración en horas
# Se verifica que haya información
q = sum(~np.isnan(Prec[Dxi[0]:Dxf[0]+1]))
if q <= len(Prec[Dxi[0]:Dxf[0]+1])*.50:
DurPrec[-1] = np.nan
TotalPrec.append(np.nan)
IntPrec.append(np.nan)
IntPrecMax.append(np.nan)
MaxPrec.append(np.nan)
Pindex.append(np.nan)
TasaPrec.append(np.nan)
DatesMax.append(np.nan)
else:
# ------------------------
# Precipitation total
# ------------------------
TotalP = np.nansum(Prec[Dxi[0]:Dxf[0]+1])
TotalPrec.append(TotalP)
# -----------------------------
# Mean Intensity precipitation
# -----------------------------
IntPrec.append(TotalP/DurPrec[-1])
if IntPrec[-1] >= 100:
DurPrec[-1] = np.nan
TotalPrec[-1] = np.nan
IntPrec[-1] = np.nan
IntPrecMax.append(np.nan)
MaxPrec.append(np.nan)
Pindex.append(np.nan)
TasaPrec.append(np.nan)
DatesMax.append(np.nan)
else:
# ------------------------
# Maximum Precipitation
# ------------------------
MaxPrec.append(np.nanmax(Prec[Dxi[0]:Dxf[0]+1]))
# -----------------------------
# Max Intensity precipitation
# -----------------------------
IntPrecMax.append(MaxPrec[-1]/(dt/60))
# ------------------------
# P Index
# ------------------------
Pindex.append(IntPrecMax[-1]/IntPrec[-1])
# ------------------------
# Dates Max
# ------------------------
x = np.where(Prec[Dxi[0]:Dxf[0]]==MaxPrec[-1])[0][-1]
DatesMax.append(DatesEv[Dxi[0]+x])
DatesMax[-1] = DUtil.Dates_str2datetime([DatesMax[-1]],Date_Format=None)[0]
# ------------------------
# Precipitation Rate
# ------------------------
TasaPrec.append((MaxPrec[-1]-Prec[Dxi[0]])/((x)*dt/60))
DatesEvMax = np.array(DatesMax)[0]
DurPrec = np.array(DurPrec)[0]
TotalPrec = np.array(TotalPrec)[0]
IntPrec = np.array(IntPrec)[0]
IntPrecMax = np.array(IntPrecMax)[0]
MaxPrec = np.array(MaxPrec)[0]
Pindex = np.array(Pindex)[0]
TasaPrec = np.array(TasaPrec)[0]
else:
for i in range(len(DatesEv)):
# Verify event data
q = sum(~np.isnan(Prec[i]))
if q <= len(DatesEv[i])*.90:
DurPrec.append(np.nan)
TotalPrec.append(np.nan)
IntPrec.append(np.nan)
IntPrecMax.append(np.nan)
MaxPrec.append(np.nan)
Pindex.append(np.nan)
TasaPrec.append(np.nan)
DatesMax.append(np.nan)
else:
# ------------------------
# Rainfall duration
# ------------------------
Dxi = np.where(DatesEv[i] == DatesEvst_Aft[i])[0]
Dxf = np.where(DatesEv[i] == DatesEvend_Aft[i])[0]
DurPrec.append((Dxf[0]-Dxi[0]+1)*dt/60) # Duración en horas
# Se verifica que haya información
q = sum(~np.isnan(Prec[i,Dxi[0]:Dxf[0]+1]))
if q <= len(Prec[i,Dxi[0]:Dxf[0]+1])*.50:
DurPrec[-1] = np.nan
TotalPrec.append(np.nan)
IntPrec.append(np.nan)
IntPrecMax.append(np.nan)
MaxPrec.append(np.nan)
Pindex.append(np.nan)
TasaPrec.append(np.nan)
DatesMax.append(np.nan)
else:
# ------------------------
# Precipitation total
# ------------------------
TotalP = np.nansum(Prec[i,Dxi[0]:Dxf[0]+1])
TotalPrec.append(TotalP)
# -----------------------------
# Mean Intensity precipitation
# -----------------------------
IntPrec.append(TotalP/DurPrec[-1])
if IntPrec[-1] >= 100:
DurPrec[-1] = np.nan
TotalPrec[-1] = np.nan
IntPrec[-1] = np.nan
IntPrecMax.append(np.nan)
MaxPrec.append(np.nan)
Pindex.append(np.nan)
TasaPrec.append(np.nan)
DatesMax.append(np.nan)
else:
# ------------------------
# Maximum Precipitation
# ------------------------
MaxPrec.append(np.nanmax(Prec[i,Dxi[0]:Dxf[0]+1]))
# -----------------------------
# Max Intensity precipitation
# -----------------------------
IntPrecMax.append(MaxPrec[-1]/(5/60))
# ------------------------
# P Index
# ------------------------
Pindex.append(IntPrecMax[-1]/IntPrec[-1])
# ------------------------
# Dates Max
# ------------------------
x = np.where(Prec[i,Dxi[0]:Dxf[0]]==MaxPrec[-1])[0][-1]
DatesMax.append(DatesEv[i,Dxi[0]+x])
DatesMax[-1] = DUtil.Dates_str2datetime([DatesMax[-1]],Date_Format=None)[0]
# ------------------------
# Precipitation Rate
# ------------------------
TasaPrec.append((MaxPrec[-1]-Prec[i,Dxi[0]])/((x)*dt/60))
DatesEvMax = np.array(DatesMax)
DurPrec = np.array(DurPrec)
TotalPrec = np.array(TotalPrec)
IntPrec = np.array(IntPrec)
IntPrecMax = np.array(IntPrecMax)
MaxPrec = np.array(MaxPrec)
Pindex = np.array(Pindex)
TasaPrec = np.array(TasaPrec)
Results = {'DurPrec':DurPrec,'TotalPrec':TotalPrec,'IntPrec':IntPrec,
'MaxPrec':MaxPrec,'DatesEvst':DatesEvst,'DatesEvend':DatesEvend,
'DatesEvMax':DatesEvMax,'Pindex':Pindex,'IntPrecMax':IntPrecMax,'TasaPrec':TasaPrec}
return Results
def ConsDaysOverOrLower(Data,Dates,Value,Comparation='Over'):
'''
DESCRIPTION:
This function calculates the number of consecutive days with
values over or lower a specific value and also gives the dates at
the beginning and end of evey season.
_______________________________________________________________________
INPUT:
+ Data: Data that needs to be counted.
+ Dates: Dates of the data, can be in datetime or string vector.
if the Dates are in a string vector it has to be in
yyyy/mm/dd.
+ Value: Value to search days over or lower.
+ Comparation: String with the comparation that is going to be
done.
It can recognize the following strings:
String | Interpretation
'Over', 'over' or '>' | >
'Lower', 'lower' or '<'| <
'>=' | >=
'<=' | <=
_______________________________________________________________________
OUTPUT:
The output is the dictionary results with the following keys:
- TotalNumDays: Vector with total number of consecutive days
above or below the value.
- TotalPrecCount: Vector with the total of values during the
different number of days, works with
precipitation, averages needs to be
determined manually.
- TotalDateB: Starting dates of the different events.
- MaxNumDays: Maximum number of consecutive days above
or below the value.
- MaxPrecCount_MaxDay: Maximum values in the maximum day.
- MaxNumDays_MaxPrec: Maximum days in the maximum values.
- DateMaxDays: Beginnig date of the maximum days count.
'''
# keys
keys = ['TotalNumDays','TotalPrecCount','TotalDateB','MaxNumDays',\
'MaxPrecCount','DateMaxDays','MaxPrecCount_MaxDay','MaxNumDays_MaxPrec']
# Determine operation
Comp = utl.Oper_Det(Comparation)
# ----------------
# Error managment
# ----------------
if Comp == -1:
return -1
# Dates Error managment
if isinstance(Dates[0],str):
DatesP = DUtil.Dates_str2datetime(Dates)
if list(DatesP)[0] == -1:
return -1
elif isinstance(DatesP[0],datetime):
Er = utl.ShowError('DaysOverOrLower','Hydro_Analysis','Dates are not in days')
return Er
else:
DatesP = Dates
if isinstance(DatesP[0],datetime):
Er = utl.ShowError('DaysOverOrLower','Hydro_Analysis','Dates are not in days')
return Er
# --------------
# Calculations
# --------------
results = dict()
results['TotalNumDays'] = [] # List with all the days
results['TotalPrecCount'] = [] # List for the total
results['TotalDateB'] = [] # List of dates
x = np.where(Comp(Data,Value))[0]
if len(x) > 1:
# Loop to calculate all the days
DaysCount = 1 # Days counter
PrecCount = Data[x[0]] # Value counter
for i in range(1,len(x)):
if x[i] == x[i-1]+1:
DaysCount += 1
PrecCount += Data[x[i]]
else:
results['TotalNumDays'].append(DaysCount)
results['TotalPrecCount'].append(PrecCount)
results['TotalDateB'].append(DatesP[x[i]-DaysCount])
DaysCount = 0
PrecCount = 0
if i == len(x)-1:
results['TotalNumDays'].append(DaysCount)
results['TotalPrecCount'].append(PrecCount)
results['TotalDateB'].append(DatesP[x[i]-DaysCount])
DaysCount = 0
PrecCount = 0
results['TotalNumDays'] = np.array(results['TotalNumDays'])
# Maximum number of days
results['MaxNumDays'] = np.max(results['TotalNumDays'])
x = np.where(results['TotalNumDays'] == results['MaxNumDays'])[0]
results['TotalPrecCount'] = np.array(results['TotalPrecCount'])
# Maximum value counter of number of days that was max
results['MaxPrecCount_MaxDay'] = np.max(results['TotalPrecCount'][x])
# Maximum value in those days
results['MaxPrecCount'] = np.max(results['TotalPrecCount'])
x = np.where(results['TotalPrecCount'] == results['MaxPrecCount'])[0]
# Maximum number of days of the maximum value
results['MaxNumDays_MaxPrec'] = np.max(results['TotalNumDays'][x])
# Beginning date of the maximum
results['TotalDateB'] = np.array(results['TotalDateB'])
xx = np.where(results['TotalNumDays'] == results['MaxNumDays'])[0]
results['DateMaxDays'] = results['TotalDateB'][xx]
else:
for ikey,key in enumerate(keys):
if ikey > 2:
results[key] = 0
else:
results[key] = np.array([0])
return results
def DaysOverOrLower(Data,Dates,Value,flagMonths=False,Comparation='Over'):
'''
DESCRIPTION:
This function calculates the days over or lower one specific value
in every year or month and year of the series.
_______________________________________________________________________
INPUT:
+ Data: Data that needs to be counted.
+ Dates: Dates of the data, can be in datetime or string vector.
if the Dates are in a string vector it has to be in
yyyy/mm/dd.
+ Value: Value to search days over or lower.
+ flagMonths: flag to know if the count would be monthly.
True: to make the count monthly.
False: to make the count annualy.
+ Comparation: String with the comparation that is going to be
done.
It can recognize the following strings:
String | Interpretation
'Over', 'over' or '>' | >
'Lower', 'lower' or '<'| <
'>=' | >=
'<=' | <=
_______________________________________________________________________
OUTPUT:
- results: number of days over or lower a value for every year or
month.
- An: Dates where the operation was made.
'''
# Determine operation
Comp = utl.Oper_Det(Comparation)
# ----------------
# Error managment
# ----------------
if Comp == -1:
return -1, -1
# Dates Error managment
if isinstance(Dates[0],str):
DatesP = DUtil.Dates_str2datetime(Dates)
if list(DatesP)[0] == -1:
return -1, -1
elif isinstance(DatesP[0],datetime):
Er = utl.ShowError('DaysOverOrLower','Hydro_Analysis','Dates are not in days')
return Er, Er
else:
DatesP = Dates
if isinstance(DatesP[0],datetime):
Er = utl.ShowError('DaysOverOrLower','Hydro_Analysis','Dates are not in days')
return Er, Er
# ----------------
# Dates managment
# ----------------
if flagMonths:
An = DUtil.Dates_datetime2str(DatesP,Date_Format='%Y/%m')
if list(An)[0] == -1:
return -1,-1
else:
An = DUtil.Dates_datetime2str(DatesP,Date_Format='%Y')
if list(An)[0] == -1:
return -1,-1
# ----------------
# Calculations
# ----------------
results = []
if flagMonths:
for i in range(DatesP[0].year,DatesP[-1].year+1):
for j in range(1,13):
if j < 10:
m = '%s/0%s' %(i,j)
else:
m = '%s/%s' %(i,j)
x = np.where(An == m)[0]
P = Data[x]
xx = np.where(Comp(P,Value))[0]
if sum(~np.isnan(P)) >= 0.70*len(P):
results.append(len(xx))
else:
results.append(np.nan)
else:
for i in range(DatesP[0].year,DatesP[-1].year+1):
x = np.where(An == str(i))[0]
P = Data[x]
xx = np.where(Comp(P,Value))[0]
if sum(~np.isnan(P)) >= 0.70*len(P):
results.append(len(xx))
else:
results.append(np.nan)
return results,An
def CompDC(Dates,V,DateI,DateE,dtm=None):
'''
DESCRIPTION:
This function completes or cut data from specific dates.
_____________________________________________________________________
INPUT:
:param Dates: Data date, it must be a string like this
'Year/month/day' the separator '/'
could be change with any character.
It must be a string vector or a date or datetime vector.
:param VC: Variable.
:param DateI: Initial Date in date or datetime format.
:param DateE: Final Date in date or datetime format.
:param dtm: Time delta for the full data, if None it
would use the timedelta from the 2 values
of the original data
_____________________________________________________________________
OUTPUT:
:return Results: A dict, Dictionary with the following results.
DatesC: Complete date string vector.
V1C: Filled data values.
DatesN: Complete date Python datetime vector.
'''
V = np.array(V)
Dates = np.array(Dates)
# ---------------------
# Error managment
# ---------------------
if isinstance(Dates[0],str) == False and isinstance(Dates[0],date) == False and isinstance(Dates[0],datetime) == False:
Er = utl.ShowError('CompD','EDSM','Bad format in dates')
raise Er
if len(Dates) != len(V):
Er = utl.ShowError('CompD','EDSM','Date and V are different length')
raise Er
if dtm != None and isinstance(dtm,timedelta) == False:
Er = utl.ShowError('CompD','EDSM','Bad dtm format')
raise Er
# ---------------------
# Dates Calculations
# ---------------------
# Original Dates
if isinstance(Dates[0],str):
DatesO = DUtil.Dates_str2datetime(Dates)
else:
DatesO = Dates
if dtm == None:
dtm = DatesO[1]-DatesO[0]
DatesN = DUtil.Dates_Comp(DateI,DateE,dtm=dtm)
# -------------------------------------------
# Data Extraction
# -------------------------------------------
# Filled data
if isinstance(V[0],str):
VC = (np.empty(len(DatesN))*np.nan).astype(str)
else:
VC = (np.empty(len(DatesN))*np.nan)
DatesN = np.array(DatesN)
DatesO = np.array(DatesO)
for iF,F in enumerate(DatesO):
x = np.where(DatesN == F)[0]
if len(x) == 1:
VC[x] = V[iF]
elif len(x) > 1:
VC[x[0]] = V[iF]
DatesC = DUtil.Dates_datetime2str(DatesN)
Results = {'DatesC':DatesC,'DatesN':DatesN,'VC':VC}
return Results
def Dates_Comp(self, DateI, DateE, dtm=timedelta(1)):
'''
DESCRIPTION:
This function creates a date or datetime vector from an initial
date to a ending date.
_______________________________________________________________________
INPUT:
:param DateI: A datetime value, Initial Date in date or datetime.
:param DateE: A datetime value, Ending Date in date or datetime.
:param dtm: A timedelta, Time delta.
_______________________________________________________________________
OUTPUT:
:return DatesC: A ndarray, Complete date or datetime vector.
'''
# ---------------------
# Constants
# ---------------------
flagH = False # flag for the hours
flagM = False # flag for the minutes
# ---------------------
# Error managment
# ---------------------
if isinstance(dtm, timedelta) == False:
Er = utl.ShowError('Dates_Comp', 'DatesUtil', 'Bad dtm format')
raise Er
if isinstance(DateI, datetime) and dtm.seconds == 0:
Er = utl.ShowError('Dates_Comp', 'DatesUtil',
'Bad time delta given')
raise Er
if isinstance(DateI, date) == False or isinstance(DateE,
date) == False:
Er = utl.ShowError('Dates_Comp', 'DatesUtil',
'Bad DateI and DateE format')
raise Er
if isinstance(DateI, datetime) and isinstance(DateE,
datetime) == False:
Er = utl.ShowError('Dates_Comp', 'DatesUtil',
'Bad DateI and DateE format')
raise Er
if isinstance(DateI, datetime) == False and isinstance(
DateE, datetime):
Er = utl.ShowError('Dates_Comp', 'DatesUtil',
'Bad DateI and DateE format')
raise Er
# ---------------------
# Generate series
# ---------------------
if isinstance(DateI, datetime):
flagH = True
yeari = DateI.year
yearf = DateE.year
monthi = DateI.month
monthf = DateE.month
DatesC = [DateI]
Date = DateI
for Y in range(yeari, yearf + 1):
for m in range(1, 13):
Fi = date(Y, m, 1)
if m == 12:
Ff = date(Y + 1, 1, 1)
else:
Ff = date(Y, m + 1, 1)
Difd = (Ff - Fi).days
for d in range(Difd):
if flagH:
Dif = dtm.seconds
if Dif < 3600:
flagM = True
if int(Dif / 60 / 60) == 0:
dtt = 1
else:
dtt = int(Dif / 60 / 60)
for h in range(0, 24, dtt):
if flagM:
# DifM = Dif/60
for M in range(0, 60, int(Dif / 60)):
if Date <= DateI or Date > DateE + dtm:
Date += dtm
else:
DatesC.append(Date)
Date += dtm
else:
if Date <= DateI or Date > DateE + dtm:
Date += dtm
else:
DatesC.append(Date)
Date += dtm
else:
if Date <= DateI or Date >= DateE + dtm:
Date += dtm
else:
DatesC.append(Date)
Date += dtm
DatesC = np.array(DatesC)
return DatesC
def CompD(Dates,V,dtm=None):
'''
DESCRIPTION:
This function takes a data series and fill the missing dates with
nan values, It would fill the entire year.
_______________________________________________________________________
INPUT:
:param Dates: A list or ndarray, Data date, it must be a string
vector or a date or datetime
vector.
:param V: A list or ndarray, Variable that wants to be
filled.
:param dtm: A list or ndarray, Time delta for the full data,
if None it would use the
timedelta from the 2 values of
the original data.
_______________________________________________________________________
OUTPUT:
:return DateC: A ndarray, Comlete date string vector.
:return VC: A ndarray, Filled data values.
:return DateN: A ndarray, Complete date Python datetime vector.
'''
V = np.array(V)
Dates = np.array(Dates)
# ---------------------
# Error managment
# ---------------------
if isinstance(Dates[0],str) == False and isinstance(Dates[0],date) == False and isinstance(Dates[0],datetime) == False:
utl.ShowError('CompD','EDSM','not expected format in dates')
if len(Dates) != len(V):
utl.ShowError('CompD','EDSM','Date and V are different length')
if dtm != None and isinstance(dtm,timedelta) == False:
utl.ShowError('CompD','EDSM','Bad dtm format')
# Eliminate the errors in February
if isinstance(Dates[0],str):
lenDates = len(Dates)
Dates2 = np.array([i[:10] for i in Dates])
for iY,Y in enumerate(range(int(Dates2[0][:4]),int(Dates2[-1][:4]))):
Fi = date(Y,2,1)
Ff = date(Y,3,1)
Dif = (Ff-Fi).days
if Dif == 28:
x = np.where(Dates2 == '%s/02/29' %(Y))
Dates = np.delete(Dates,x)
V = np.delete(V,x)
x = np.where(Dates2 == '%s/02/30' %(Y))
Dates = np.delete(Dates,x)
V = np.delete(V,x)
# ---------------------
# Dates Calculations
# ---------------------
# Original Dates
if isinstance(Dates[0],str):
DatesO = DUtil.Dates_str2datetime(Dates)
else:
DatesO = Dates
if dtm == None:
dtm = DatesO[1]-DatesO[0]
# Complete Dates
if isinstance(DatesO[0],datetime):
DateI = datetime(DatesO[0].year,1,1,0,0)
DateE = datetime(DatesO[-1].year,12,31,23,59)
DatesN = DUtil.Dates_Comp(DateI,DateE,dtm=dtm)
else:
DateI = date(DatesO[0].year,1,1)
DateE = date(DatesO[-1].year,12,31)
DatesN = DUtil.Dates_Comp(DateI,DateE,dtm=dtm)
# Filled data
VC = np.empty(len(DatesN))*np.nan
DatesN = np.array(DatesN)
DatesO = np.array(DatesO)
V = np.array(V)
# x = DatesN.searchsorted(DatesO)
x = np.searchsorted(DatesN,DatesO)
try:
VC[x] = V
except ValueError:
VC = np.array(['' for i in range(len(DatesN))]).astype('<U20')
VC[x] = V
DatesC = DUtil.Dates_datetime2str(DatesN)
Results = {'DatesC':DatesC,'DatesN':DatesN,'VC':VC}
return Results
def Dates_str2datetime(self, Dates, Date_Format=None, flagQuick=False):
'''
DESCRIPTION:
From a string type vector the function converts the dates to
python date or datetime data from a given format, if no date
format is specified the function looks up for a fitting format.
_______________________________________________________________________
INPUT:
+ Dates: String date vector that needs to be changed to date or
datetime vector.
+ Dates_Format: Format of the dates given, it must be given in
datetime string format like %Y/%m/%d %H%M.
_______________________________________________________________________
OUTPUT:
- DatesP: Python date or datetime format vector.
'''
# ----------------
# Error managment
# ----------------
if isinstance(Dates[0], str) == False:
Er = utl.ShowError('Dates_str2datetime', 'DatesUtil',
'Bad Dates format given, not in string format')
raise TypeError
# -------------------------
# Temporal verification
# -------------------------
lenDates = len(Dates[0])
flagHour = False
if lenDates > 10:
flagHour = True
# -------------------------
# Date_Format Verification
# -------------------------
if Date_Format == None:
if flagHour:
Date_Formats = self.DateTime_Formats
else:
Date_Formats = self.Date_Formats
else:
Date_Formats = [Date_Format]
# -------------------------
# Transformation
# -------------------------
for iF, F in enumerate(Date_Formats):
try:
if flagQuick:
if flagHour:
DatesP2 = np.array([
datetime(int(i[:4]), int(i[5:7]), int(i[8:10]),
int(i[11:13]), int(i[13:15]))
for i in Dates
])
else:
DatesP2 = np.array([
datetime(int(i[:4]), int(i[5:7]), int(i[8:10]), 0,
0) for i in Dates
])
else:
DatesP2 = np.array(
[datetime.strptime(i, F) for i in Dates])
break
except:
if iF == len(Date_Formats) - 1:
Er = utl.ShowError('Dates_str2datetime', 'DatesUtil',
'Bad date format, change format')
raise TypeError
else:
continue
# -------------------------
# Changing Dates
# -------------------------
if flagHour == False:
DatesP = np.array([i.date() for i in DatesP2])
else:
DatesP = DatesP2
return DatesP
def FF(self, xdata, ydata, F='lineal', alpha=0.05, flagParabolic=False):
'''
DESCRIPTION:
This function takes x and y data and makes a fitting of the data with
the function that wants to be added.
_______________________________________________________________________
INPUT:
:param xdata: A ndArray, Data in the x axis.
:param ydata: A ndArray, Data in the y axis.
:param F: A str, Function that wants to be fitted,
by default is 'lineal'.
:param alpha: A float, significance level.
:param flagParabolic: A boolean, flag to adjust Parabolic in
automatic.
The functions that are defaulted to fit are the following:
_______________________________________________________________________
OUTPUT:
This function return a dictionary with the following data:
- Coef: Ceofficients
- perr: Standard deviation errors of the parameters.
- R2: Coefficient of determination.
'''
# --------------------------------
# Error Managment and Parameters
# --------------------------------
if isinstance(F, str) == False and F != 0:
utl.ShowError('FF', 'CFitting',
'Given parameter F is not on the listed functions.')
keys = list(self.Adj)
if F == '' or F == 0:
flagfitbest = True
else:
key = F.lower()
flagfitbest = False
try:
fun = self.Adj[key]
except KeyError:
utl.ShowError(
'FF', 'CFitting',
'Given parameter F is not on the listed functions.')
# -----------------
# Calculations
# -----------------
X, Y = DM.NoNaN(xdata, ydata, False)
Results = dict()
if flagfitbest:
CoefT = dict()
perrT = dict()
keysT = []
R2T = []
keys2 = []
for ikey, key in enumerate(keys):
if not (flagParabolic) and key == 'parabolic':
continue
try:
keys2.append(key)
fun = self.Adj[key]
# Fitting
Coef, pcov = curve_fit(fun, X, Y)
# R2 calculations
ss_res = np.dot((Y - fun(X, *Coef)), (Y - fun(X, *Coef)))
ymean = np.mean(Y)
ss_tot = np.dot((Y - ymean), (Y - ymean))
R2T.append(1 - (ss_res / ss_tot))
perrT[key] = np.sqrt(np.diag(pcov))
CoefT[key] = Coef
except RuntimeError:
print('WARNING: Cannot fit a', key)
perrT[key] = np.nan
CoefT[key] = np.nan
R2T.append(np.nan)
continue
# Verify the maximum R^2
x = np.where(np.array(R2T) == np.nanmax(np.array(R2T)))[0]
if len(x) > 1:
x = x[0]
key = np.array(keys2)[x][0]
Results['Coef'] = CoefT[key]
Results['perr'] = perrT[key]
Results['R2'] = np.array(R2T)[x][0]
Results['Functionkey'] = key
Results['Function'] = self.Adj[key]
Results['FunctionEq'] = self.AdjF[key]
else:
# Fitting
Coef, pcov = curve_fit(fun, X, Y)
ss_res = np.dot((Y - fun(X, *Coef)), (Y - fun(X, *Coef)))
perr = np.sqrt(np.diag(pcov))
ymean = np.mean(Y)
ss_tot = np.dot((Y - ymean), (Y - ymean))
R2 = 1 - (ss_res / ss_tot)
Results['Coef'] = Coef
Results['perr'] = perr
Results['R2'] = R2
Results['Functionkey'] = key
Results['Function'] = fun
Results['FunctionEq'] = self.FunctionsEqstr(self.AdjF[key], Coef)
# Confidence intervals of the parameters
n = len(Y) # Number of data
p = len(Results['Coef']) # Number of parameters
dof = max(0, n - p) # Number of degrees of freedom
# Student-t value for the dof and confidence level
tval = t.ppf(1.0 - alpha / 2.0, dof)
Results['ConInt'] = []
for i, p, var in zip(range(n), Results['Coef'], np.diag(pcov)):
sigma = var**0.5
Results['ConInt'].append([p - sigma * tval, p + sigma * tval])
# ---------------------
# Errors
# ---------------------
# Estimation Error
SS = []
VC = Results['Function'](X, *Results['Coef'])
for iy, y in enumerate(VC):
SS.append((Y[iy] - y)**2)
EErr = np.sqrt((1 / (n - 2)) * np.sum(np.array(SS)))
Results['EErr'] = EErr
# RSME (Root Mean Square Error)
SS = []
VC = Results['Function'](X, *Results['Coef'])
for iy, y in enumerate(VC):
SS.append((y - Y[iy])**2)
RSME = np.sqrt((1 / (n)) * np.sum(np.array(SS)))
Results['RSME'] = RSME
# MBE (Mean Bias Error)
SS = []
VC = Results['Function'](X, *Results['Coef'])
for iy, y in enumerate(VC):
SS.append((y - Y[iy]))
RSME = np.sqrt((1 / (n)) * np.sum(np.array(SS)))
Results['MBE'] = RSME
# MPE (Mean Percentage Error)
SS = []
VC = Results['Function'](X, *Results['Coef'])
for iy, y in enumerate(VC):
SS.append(((y - Y[iy]) / Y[iy]) * 100)
RSME = np.sqrt((1 / (n)) * np.sum(np.array(SS)))
Results['MPE'] = RSME
return Results
def LoadData(self, Station=None, flagComplete=True, dt=5):
'''
DESCRIPTION:
This function loads the data of a station and compiles
it in a dictionary.
___________________________________________________________________
INPUT:
:param Station: A str, List with the stations that would
be extracted.
:param flagComplete: A boolean, flag to determine if
completes the data
'''
self.flagComplete = flagComplete
self.Station = Station
# -------------------------
# Error Managment
# -------------------------
if not (isinstance(Station, list)) and Station != None and not (
isinstance(Station, str)):
Er = utl.ShowError('OpenWundergrounds', 'LoadData',
'Erroneus type for parameter Station')
raise TypeError
# -------------------------
# Stations
# -------------------------
if Station == None:
Station = self.Stations[0]
elif isinstance(Station, list):
Station = Station[0]
# -------------------------
# Parameters
# -------------------------
DataBase = {
'DataBaseType': 'txt',
'deli': self.deli,
'colStr': None,
'colData': None,
'row_skip': 1,
'flagHeader': True,
'rowH': 0,
'row_end': 0,
'str_NaN': '',
'num_NaN': None,
'dtypeData': float
}
LabelsH = ['H', 'maxH', 'minH']
LabelsD = ['D', 'NF', 'NNF', 'maxD', 'M', 'minM', 'maxM', 'minD']
self.LabelsH = LabelsH
self.LabelsD = LabelsD
# -------------------------
# Verify Headers
# -------------------------
# Headers
Headers = np.genfromtxt(self.Arch[Station][0],
dtype=str,
skip_header=0,
delimiter=self.deli,
max_rows=1)
# Verify colStr data
colStr = []
LabStrNo = []
LabStrYes = []
for lab in self.LabelsStr:
x = np.where(Headers == lab)[0]
if len(x) > 0:
colStr.append(x[0])
LabStrYes.append(lab)
else:
LabStrNo.append(lab)
DataBase['colStr'] = tuple(colStr)
# Verify colData data
colData = []
LabDataNo = []
LabDataYes = []
for lab in self.LabelsData:
x = np.where(Headers == lab)[0]
if len(x) > 0:
colData.append(x[0])
LabDataYes.append(lab)
else:
LabDataNo.append(lab)
self.LabStrYes = LabDataYes
DataBase['colData'] = tuple(colData)
# -------------------------
# Extract information
# -------------------------
EM = EMSD()
for iar, ar in enumerate(self.Arch[Station]):
try:
R = EM.Open_Data(ar, DataBase=DataBase)
except ValueError:
print('Error document data:', ar)
continue
if iar == 0:
Data = R
else:
for iLab, Lab in enumerate(LabStrYes):
Data[Lab] = np.hstack((Data[Lab], R[Lab]))
for iLab, Lab in enumerate(LabDataYes):
try:
Data[Lab] = np.hstack((Data[Lab], R[Lab]))
except KeyError:
Data[Lab] = np.hstack(
(Data[Lab],
np.empty(R[LabDataYes[0]].shape) * np.nan))
# for Lab in LabStrNo:
# R[Lab] = np.array(['nan' for i in len(R['Time'])])
# for Lab in LabDataNo:
# R[Lab] = np.array([np.nan for i in len(R['Time'])])
# DatesS = [i[:16] for i in Data['Time']]
DatesS = Data['Time']
Dates = DUtil.Dates_str2datetime(DatesS,
Date_Format='%Y-%m-%d %H:%M:%S')
Data.pop('Time', None)
# -------------------------
# Data Completion
# -------------------------
LabelsHmat = []
LabelsDmat = []
# Data in years
DataC = dict()
DataCC = dict()
# Se llenan los datos
DataH = dict() # Datos Horarios
DataD = dict() # Datos Diarios
self.DatesC = dict()
self.DatesD = dict() # Fechas diarias
for iLab, Lab in enumerate(LabDataYes):
VC = DMan.CompD(Dates, Data[Lab], dtm=timedelta(0, 60))
# Precipitation corrected
if Lab == 'HourlyPrecipMM':
VC['VC'] = VC['VC'] * 5 / 60
if iLab == 0:
DatesC = VC['DatesC']
DataC[self.LabDataSave[Lab]] = VC['VC']
# Se pasa la información a cada 5 minutos
DatesCC, DatesCN, DataCC[self.LabDataSave[Lab]] = DMan.Ca_E(
DatesC,
DataC[self.LabDataSave[Lab]],
dt=dt,
escala=-1,
op=self.LabDataOper[Lab],
flagNaN=False)
# Data Eliminations
if self.ElimOver[self.LabDataSave[Lab]] != None:
DataCC[self.LabDataSave[Lab]][
DataCC[self.LabDataSave[Lab]] > self.ElimOver[
self.LabDataSave[Lab]]] = np.nan
if self.ElimLow[self.LabDataSave[Lab]] != None:
DataCC[self.LabDataSave[Lab]][
DataCC[self.LabDataSave[Lab]] < self.ElimLow[
self.LabDataSave[Lab]]] = np.nan
# Se convierten los datos
DatesC2, DatesNC2, VC2 = EM.Ca_EC(Date=DatesCC,
V1=DataCC[self.LabDataSave[Lab]],
op=self.LabDataOper1[Lab],
key=None,
dtm=dt,
op2=self.LabDataOper2[Lab],
op3=self.LabDataOper3[Lab])
for LabH in LabelsH:
DataH[self.LabDataSave[Lab] + LabH] = VC2[LabH]
LabelsHmat.append(self.LabDataSave[Lab] + LabH)
for LabD in LabelsD:
DataD[self.LabDataSave[Lab] + LabD] = VC2[LabD]
LabelsDmat.append(self.LabDataSave[Lab] + LabD)
self.DataH = DataH
self.DataD = DataD
self.DatesH = DatesC2['DateH']
self.DatesD['DatesD'] = DatesC2['DateD']
self.DatesD['DatesM'] = DatesC2['DateM']
self.DataCC = DataCC
self.DatesCC = DatesCC
self.DatesCN = DatesCN
self.DatesNC2 = DatesNC2['DateMN']
self.LabelsHmat = LabelsHmat
self.LabelsDmat = LabelsDmat
return
