def get3DVar(oFileData, sVarName, iVarTSlice=None, bSetAutoMask=True):
if not bSetAutoMask:
Exc.getExc(
' =====> WARNING: auto_mask is set to false! (lib_data_io_netcdf)',
2, 1)
oFileData.set_auto_mask(False)
a3dVarName_IN = oFileData.variables[sVarName][:]
if iVarTSlice is None:
oVarName_OUT = np.zeros([
a3dVarName_IN.shape[1], a3dVarName_IN.shape[2],
a3dVarName_IN.shape[0]
])
for iT in range(0, a3dVarName_IN.shape[0]):
a2dVarName_IN = np.transpose(np.rot90(a3dVarName_IN[iT, :, :], -1))
oVarName_OUT[:, :, iT] = a2dVarName_IN
# a2dVarName_IN[a2dVarName_IN < -900] = np.nan
# Debug
# plt.figure(1); plt.imshow(a2dVarName_IN); plt.colorbar(); plt.clim(-10,40);
# plt.show()
else:
oVarName_OUT = np.transpose(
np.rot90(a3dVarName_IN[iVarTSlice, :, :], -1))
return oVarName_OUT
def getVarGeo_LAMI_2i(oFileHandle, iIdx=0):
# Check method
a2dGeoY = None
a2dGeoX = None
try:
# Get file message at index [iIdx]
oFileMsg = oFileHandle()[iIdx]
# Get geographical data
try:
# Method to get lats and lons
[a2dGeoY, a2dGeoX] = oFileMsg.latlons()
except BaseException:
# Get data dimension(s)
if oFileMsg.has_key('Ni') and oFileMsg.has_key('Nj'):
iRows = oFileMsg.Ni
iCols = oFileMsg.Nj
else:
[iCols, iRows] = oFileMsg.values.shape
# Get lats and lons using key(s)
a2dGeoX = np.reshape(oFileMsg['longitudes'], [iCols, iRows])
a2dGeoY = np.reshape(oFileMsg['latitudes'], [iCols, iRows])
except BaseException:
# Exit status with warning
Exc.getExc(' -----> WARNING: in getVarAttr_LAMI_2i an exception occurred (lib_data_io_grib)', 2, 1)
return a2dGeoX, a2dGeoY
def getTimeNow(sTimeNow='', sTimeRefType='local'):
if sTimeNow is None:
sTimeNow = ''
if not sTimeNow == '':
[sTimeNow, sTimeFormat] = correctTimeLength(sTimeNow)
elif sTimeNow == '':
if sTimeRefType == 'local':
sTimeNow = getTimeLocal()
elif sTimeRefType == 'GMT':
sTimeNow = getTimeGMT()
else:
Exc.getExc(
' =====> WARNING: sTimeTypeRef is not defined correctly! sTimeNow initialized as local time!',
2, 1)
sTimeNow = getTimeLocal()
[sTimeNow, sTimeFormat] = correctTimeLength(sTimeNow)
else:
sTimeFormat = None
Exc.getExc(
' =====> ERROR: sTimeNow format is unknown. Please check your time string!',
1, 1)
return sTimeNow, sTimeFormat
def correctTimeLength(sTime):
if len(sTime) == 12:
sTimeUpd = sTime
elif len(sTime) >= 8 and len(sTime) < 12:
iTimeLength = len(sTime)
iTimeLessDigits = 12 - iTimeLength
sTimeLessFormat = '0' * iTimeLessDigits
sTimeUpd = sTime + sTimeLessFormat
elif len(sTime) > 12:
Exc.getExc(
' =====> ERROR: sTime has not allowed length (greater then 12 char). sTime cannot defined!',
1, 1)
elif len(sTime) < 8:
Exc.getExc(
' =====> ERROR: sTime has not allowed length (less then 8 char). sTime cannot defined!',
1, 1)
sTimeUpdFormat = defineTimeFormat(sTimeUpd)
# Check time format definition
checkTimeFormat(sTimeUpd, sTimeUpdFormat)
return sTimeUpd, sTimeUpdFormat
def checkTimeFormat(sTime, sTimeFormat=sTimeFormat_Default):
try:
datetime.datetime.strptime(sTime, sTimeFormat)
except BaseException:
Exc.getExc(
' =====> ERROR: sTime has not correct format. sTime cannot defined!',
1, 1)
def importFileDict(sFileName, DataName='Data'):
if exists(sFileName):
try:
if version_info >= (3, 0):
oFileData = SourceFileLoader(DataName, sFileName).load_module()
elif version_info < (3, 0):
oFileData = load_source(DataName, '', open(sFileName))
oFileDict = {}
for sFileVar in vars(oFileData):
if not sFileVar.startswith('__'):
oFileValue = getattr(oFileData, sFileVar)
oFileDict[sFileVar] = oFileValue
else:
pass
bFileData = True
except BaseException:
Exc.getExc(' =====> WARNING: read file ' + sFileName + ' FAILED!',
2, 1)
oFileDict = None
bFileData = False
else:
Exc.getExc(' =====> WARNING: file ' + sFileName + ' NOT FOUND!', 2, 1)
oFileDict = None
bFileData = False
return oFileDict, bFileData
def findTimeClosest(sTime_Ref, oTime_HH=None):
# Create datetime object
oTime_Ref = datetime.datetime.strptime(sTime_Ref, sTimeFormat_Default)
# Closest time using hours
if oTime_HH is not None:
a1oTime_Delta = []
for sTime_HH_Step in oTime_HH:
oTime_Step = oTime_Ref.replace(hour=int(sTime_HH_Step))
oTime_Delta = oTime_Ref - oTime_Step
if oTime_Delta > datetime.timedelta(seconds=1):
a1oTime_Delta.append(oTime_Delta)
oTime_Delta_Min = min(a1oTime_Delta)
iTime_Delta_Index = a1oTime_Delta.index(oTime_Delta_Min)
sTime_HH_Select = oTime_HH[iTime_Delta_Index]
oTime_Select = oTime_Ref.replace(hour=int(sTime_HH_Select))
sTime_Select = oTime_Select.strftime(sTimeFormat_Default)
return sTime_Select
else:
Exc.getExc(
' =====> WARNING: some arguments to find closest time are not defined! Check your data!',
2, 1)
return sTime_Ref
def streamProcess(sOut=None, sErr=None):
if sOut is None and sErr is None:
return True
else:
Exc.getExc(
' =====> WARNING: error(s) occurred during process execution!', 2,
1)
return False
def getVar3D_LAMI_2i(oFileHandle, oVarComp=None):
# Check method
oVarData = None
try:
# List conversion for string arguments of variable component(s)
if isinstance(oVarComp, str) and (oVarComp is not None):
oVarComp = [oVarComp]
if oVarComp is None:
oVarComp = [None]
# Iterate over variable(s) component(s)
oVarData = {}
for sVarComp in oVarComp:
if sVarComp is None:
oFileMsg = oFileHandle()
else:
try:
oFileMsg = oFileHandle.select(nameECMF=sVarComp)
except BaseException:
Exc.getExc(
' -----> WARNING: selected variable [' +
sVarComp + '] is not in grib message (lib_data_io_grib)', 2, 1)
oFileMsg = None
# Iterate over message(s)
for iVarID, oVarMsg in enumerate(oFileMsg):
# Get variable name
sVarName = oVarMsg.nameECMF
if sVarName not in oVarData:
oVarData[sVarName] = None
a2dVarData_Raw = oVarMsg.values
if ma.is_masked(a2dVarData_Raw):
a2dVarData = a2dVarData_Raw.data
else:
a2dVarData = a2dVarData_Raw
if oVarData[sVarName] is None:
a3dVarData = np.reshape(a2dVarData, [a2dVarData.shape[0], a2dVarData.shape[1], 1])
else:
a3dVarData = oVarData[sVarName]
a3dVarData = np.dstack((a3dVarData, a2dVarData))
oVarData[sVarName] = a3dVarData
except BaseException:
# Exit status with warning
Exc.getExc(' -----> WARNING: in getVar3D_LAMI_2i an exception occurred (lib_data_io_grib)', 2, 1)
return oVarData
def get1DVar(oFileData, sVarName):
try:
a1dVarName_OUT = oFileData.variables[sVarName][:]
except RuntimeError:
a1dVarName_OUT = None
Exc.getExc(
' =====> ERROR: in reading variable 1d. Check variable name! (lib_data_io_netcdf)',
1, 1)
return a1dVarName_OUT
def checkZip(sFileName):
try:
oFileTest = bz2.BZ2File(sFileName).read()
bFileCheck = True
except BaseException:
Exc.getExc(
' -----> WARNING: errors occurred in checking file integrity (lib_data_zip_bz2)',
2, 1)
bFileCheck = False
return bFileCheck
def unzipFile(oFile_IN=None, oFile_OUT=None):
if oFile_IN and oFile_OUT:
try:
oDecompressData = oFile_IN.read()
oFile_OUT.write(oDecompressData)
except BaseException:
Exc.getExc(
' -----> WARNING: errors occurred in unzipping file (lib_data_zip_bz2)',
2, 1)
else:
Exc.getExc(' -----> ERROR: in unzip file (lib_data_zip_bz2)', 1, 1)
def openFile(sFileName, sFileMode):
import h5py
try:
# Open file
oFile = h5py.File(sFileName, sFileMode)
return oFile
except IOError as oError:
Exc.getExc(
' -----> ERROR: in open file (lib_data_io_hdf5)' + ' [' +
str(oError) + ']', 1, 1)
def openFile(sFileName):
# Check method
try:
# Open file
oFile = gdal.Open(sFileName)
return oFile
except IOError as oError:
Exc.getExc(
' -----> ERROR: in open file (lib_data_io_hdf4)' + ' [' +
str(oError) + ']', 1, 1)
def openFile(sFileName, sFileMode):
try:
if 'r' in sFileMode:
oFile = open(sFileName, sFileMode + 'b')
elif 'w' in sFileMode:
oFile = open(sFileName, sFileMode + 'b')
return oFile
except IOError as oError:
Exc.getExc(
' -----> ERROR: in open file (lib_data_io_pickle)' + ' [' +
str(oError) + ']', 1, 1)
def openFile(sFileName, sFileMode='r'):
# Open file
try:
if sFileMode == 'r':
oFile = pygrib.open(sFileName)
return oFile
else:
Exc.getExc(' -----> WARNING: open file in write/append mode not available (lib_data_io_grib)', 2, 1)
return None
except IOError as oError:
Exc.getExc(' -----> ERROR: in open file (lib_data_io_grib)' + ' [' + str(oError) + ']', 1, 1)
def getTimeStep(sTimeIN=None, iTimeDelta=3600, iTimeStep=0):
if sTimeIN:
sTimeFormat = defineTimeFormat(sTimeIN)
oTimeIN = datetime.datetime.strptime(sTimeIN, sTimeFormat)
oTimeOUT = oTimeIN + datetime.timedelta(seconds=iTimeStep * iTimeDelta)
sTimeOUT = oTimeOUT.strftime(sTimeFormat)
else:
sTimeOUT = None
sTimeFormat = None
Exc.getExc(
" =====> ERROR: sTimeIN not defined! sTimeOUT cannot calculated!",
1, 1)
return sTimeOUT, sTimeFormat
def convertTimeLOCxGMT(sTime_IN=None, iTimeDiff=0):
if sTime_IN:
sTimeFormat_IN = defineTimeFormat(sTime_IN)
sTimeFormat_OUT = sTimeFormat_IN
oTime_IN = datetime.datetime.strptime(sTime_IN, sTimeFormat_IN)
oTime_IN = oTime_IN.replace(minute=0, second=0, microsecond=0)
oTime_OUT = oTime_IN + datetime.timedelta(seconds=iTimeDiff * 3600)
sTime_OUT = oTime_OUT.strftime(sTimeFormat_OUT)
else:
Exc.getExc(
" =====> ERROR: sTime_IN not defined! sTime_OUT cannot calculated!",
1, 1)
return sTime_OUT
def getTimeFrom(sTimeTo=None, iTimeDelta=3600, iTimeStep=0):
if sTimeTo:
sTimeFormat = defineTimeFormat(sTimeTo)
oTimeTo = datetime.datetime.strptime(sTimeTo, sTimeFormat)
oTimeFrom = oTimeTo - datetime.timedelta(seconds=iTimeStep *
iTimeDelta)
sTimeFrom = oTimeFrom.strftime(sTimeFormat)
else:
sTimeFrom = None
sTimeFormat = None
Exc.getExc(
" =====> ERROR: sTimeTo not defined! sTimeFrom cannot calculated!",
1, 1)
return sTimeFrom, sTimeFormat
def getFileMessage(oFile):
try:
iFM = oFile.messages
if iFM == 0:
a1iFM = None
Exc.getExc(' -----> WARNING: no message in GRIB file! Check your input file!', 2, 1)
else:
a1iFM = np.linspace(1, iFM, iFM, endpoint=True)
except BaseException:
a1iFM = None
Exc.getExc(' -----> WARNING: no message in GRIB file! Check your input file!', 2, 1)
return a1iFM
def checkProcess(sCLine=None,
sCPath=None,
iTimeError=0.001,
iTimeWait=1,
iTimePool=0.1):
# Info checking starting
oLogStream.info(' -----> Process checking: ' + sCLine + ' ... ')
# Checking command before execution (to find errors) with a given time
os.chdir(sCPath)
oProcess = subprocess.Popen(sCLine,
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
oProcess.poll()
# Check process to control library or memory error(s)
while oProcess.returncode is None and iTimeWait > 0:
time.sleep(iTimePool)
iTimeWait -= iTimePool
oProcess.poll()
# Condition to define if process checking is fine or not
if iTimeWait <= 0: # process request stops for time out
# Checking process for run until time-out
iPID = oProcess.pid + 1 # There is a difference between python-pid and htop-pid (+1)
os.kill(iPID, signal.SIGTERM)
# Information process for run until time-out
oLogStream.info(' -----> Process checking: ' + sCLine +
' ... OK [run time-out]')
elif iTimeWait < iTimeError: # process request stops for errors
# Checking stops for error(s)
oFrameInfo = getframeinfo(currentframe())
# Information about stop for error(s)
oLogStream.info(' -----> Process checking: ' + sCLine + ' ... FAILED!')
Exc.getExc(
' =====> ERROR: process checking failed for library or memory errors!',
1, 1, oFrameInfo.filename, oFrameInfo.lineno)
else:
# Information process for run closing
oLogStream.info(' -----> Process checking: ' + sCLine +
' ... OK [run ending]')
def getVar3D_ALL(oFile, sVarComp=None):
# Check method
try:
bVarWarn = False
oVarData = {}
for oVar in oFile:
# Get variable name
sVarName = oVar.name
#if not isinstance(sVarName, str):
# sVarName = sVarName.encode('UTF-8')
#sVarName = sVarName.replace(' ', '_').lower()
if (sVarComp == sVarName) or (sVarComp is None):
if sVarName not in oVarData:
oVarData[sVarName] = None
a2dVarData_Raw = oVar.values
if ma.is_masked(a2dVarData_Raw):
a2dVarData = a2dVarData_Raw.data
else:
a2dVarData = a2dVarData_Raw
if oVarData[sVarName] is None:
a3dVarData = np.reshape(a2dVarData, [a2dVarData.shape[0], a2dVarData.shape[1], 1])
else:
a3dVarData = oVarData[sVarName]
a3dVarData = np.dstack((a3dVarData, a2dVarData))
oVarData[sVarName] = a3dVarData
else:
# Exit status with warning
if bVarWarn is False:
Exc.getExc(
' -----> WARNING: selected variable [' + sVarComp +
'] is not in grib message (lib_data_io_grib)', 2, 1)
bVarWarn = True
return oVarData
except BaseException:
# Exit status with warning
Exc.getExc(' -----> WARNING: in getting 3d variable function (lib_data_io_grib)', 2, 1)
def get2DVar(oFileData, sVarName, bSetAutoMask=True):
if not bSetAutoMask:
Exc.getExc(
' =====> WARNING: auto_mask is set to false! (lib_data_io_netcdf)',
2, 1)
oFileData.set_auto_mask(False)
try:
a2dVarName_IN = oFileData.variables[sVarName][:]
a2dVarName_OUT = np.transpose(np.rot90(a2dVarName_IN, -1))
except RuntimeError:
a2dVarName_OUT = None
Exc.getExc(
' =====> ERROR: in reading variable 2d. Check variable name! (lib_data_io_netcdf)',
1, 1)
return a2dVarName_OUT
def openFile(sFileName, sFileMode, sFileFormat='NETCDF4'):
try:
from netCDF4 import Dataset as Dataset
except BaseException:
from scipy.io.netcdf import netcdf_file as Dataset
Exc.getExc(
' =====> WARNING: NetCDF module import from scipy (usually imported from netcdf-python)!',
2, 1)
try:
# NetCDF type: NETCDF3_CLASSIC , NETCDF3_64BIT , NETCDF4_CLASSIC , NETCDF4
oFile = Dataset(sFileName, sFileMode, format=sFileFormat)
return oFile
except IOError as oError:
Exc.getExc(
' =====> ERROR: in open file (lib_data_io_netcdf)' + ' [' +
str(oError) + ']', 1, 1)
def getVarGeo(oFile, iIndex=1):
# Get geographical information
try:
# Get geographical data
[a2dGeoY, a2dGeoX] = oFile[iIndex].latlons()
# Check geographical orientation
[a2dVarGeoX, a2dVarGeoY, bVarCorrOrient] = checkVarOrient(a2dGeoX, a2dGeoY)
# Store geographical information
a1oVarGeo = {'Longitude': a2dGeoX, 'Latitude': a2dGeoY, 'CorrOrient': bVarCorrOrient}
except BaseException:
Exc.getExc(' -----> WARNING: error in extracting file geographical data! Check input file!', 2, 1)
a1oVarGeo = None
# Return variable(s)
return a1oVarGeo
def openFile(sFileName, sFileMode=None, sFileDelimiter=','):
# Method to open an object to store features of csv dataset
try:
oFile = ObjCSV()
if exists(sFileName):
oFile.bFileData = True
else:
oFile.bFileData = False
oFile.sFileMode = sFileMode
oFile.sFileName = sFileName
oFile.sFileDelimiter = sFileDelimiter
return oFile
except IOError as oError:
Exc.getExc(
' -----> ERROR: in open file (lib_data_io_csv)' + ' [' +
str(oError) + ']', 1, 1)
def openZip(sFileName_IN, sFileName_OUT, sZipMode):
# Check method
try:
# Open file
oFile_IN = None
oFile_OUT = None
if sZipMode == 'z': # zip mode
oFile_IN = open(sFileName_IN, 'rb')
oFile_OUT = bz2.BZ2File(sFileName_OUT, 'wb')
elif sZipMode == 'u': # unzip mode
oFile_IN = bz2.BZ2File(sFileName_IN)
oFile_OUT = open(sFileName_OUT, "wb")
# Pass file handle(s)
return oFile_IN, oFile_OUT
except IOError as oError:
Exc.getExc(
' -----> ERROR: in open file (lib_data_zip_bz2)' + ' [' +
str(oError) + ']', 1, 1)
def getVars(oFile, sDsetValueKey='values', sDsetAttrKey='parameters'):
oFileDict = {}
for sDataSet in oFile:
oDataSet = oFile[sDataSet]
oFileDict[sDataSet] = {}
try:
oFileDict[sDataSet][sDsetValueKey] = {}
oFileDict[sDataSet][sDsetAttrKey] = {}
oFileDict[sDataSet][sDsetValueKey] = oDataSet.value
for oItem in oDataSet.attrs.keys():
oValue = oDataSet.attrs[oItem]
oFileDict[sDataSet][sDsetAttrKey][oItem] = {}
oFileDict[sDataSet][sDsetAttrKey][oItem] = oValue
except BaseException:
Exc.getExc(
' -----> Warning: impossible to get data values or attributes! Check your file! (lib_data_io_hdf5)',
2, 1)
return oFileDict
def getVar2D_ALL(oFile, iIndex=None):
# Check method
try:
if iIndex:
# Read variable using Data index
oVar = oFile[iIndex]
else:
oVar = oFile
a2dVarData_Raw = oVar.values
if ma.is_masked(a2dVarData_Raw):
a2dVarData = a2dVarData_Raw.data
else:
a2dVarData = a2dVarData_Raw
return a2dVarData
except BaseException:
# Exit status with error
Exc.getExc(' -----> WARNING: in getting 2d variable function (lib_data_io_grib)', 2, 1)
def getFileAttr(oFile, iIndex=None, oVarKeyNA=None):
# Default argument
if oVarKeyNA is None:
oVarKeyNA = ['distinctLatitudes', 'distinctLongitudes',
'values', 'latLonValues',
'latitudes', 'longitudes']
if iIndex is None:
oVarHandle = oFile
else:
oVarHandle = oFile[iIndex]
# Attributes
oVarAttrs = {}
for sVarKey in oVarHandle.keys():
sVarKey = sVarKey.strip().encode('UTF-8')
sVarKey = sVarKey.decode('utf-8')
if sVarKey not in oVarKeyNA:
try:
if sVarKey == 'validDate':
oVarValue = oVarHandle.validDate
elif sVarKey == 'analDate':
oVarValue = oVarHandle.analDate
else:
oVarValue = oVarHandle[sVarKey]
oVarAttrs.update({sVarKey: oVarValue})
except BaseException:
Exc.getExc(' -----> WARNING: file key ' +
sVarKey + ' not correctly retrieved Check input file!', 2, 1)
else:
pass
return oVarAttrs
