def get3DVar(oFileData, sVarName, iVarTSlice=None, bSetAutoMask=True): if not bSetAutoMask: Exc.getExc( ' =====> WARNING: auto_mask is set to false! (lib_data_io_netcdf)', 2, 1) oFileData.set_auto_mask(False) a3dVarName_IN = oFileData.variables[sVarName][:] if iVarTSlice is None: oVarName_OUT = np.zeros([ a3dVarName_IN.shape[1], a3dVarName_IN.shape[2], a3dVarName_IN.shape[0] ]) for iT in range(0, a3dVarName_IN.shape[0]): a2dVarName_IN = np.transpose(np.rot90(a3dVarName_IN[iT, :, :], -1)) oVarName_OUT[:, :, iT] = a2dVarName_IN # a2dVarName_IN[a2dVarName_IN < -900] = np.nan # Debug # plt.figure(1); plt.imshow(a2dVarName_IN); plt.colorbar(); plt.clim(-10,40); # plt.show() else: oVarName_OUT = np.transpose( np.rot90(a3dVarName_IN[iVarTSlice, :, :], -1)) return oVarName_OUT
def getVarGeo_LAMI_2i(oFileHandle, iIdx=0): # Check method a2dGeoY = None a2dGeoX = None try: # Get file message at index [iIdx] oFileMsg = oFileHandle()[iIdx] # Get geographical data try: # Method to get lats and lons [a2dGeoY, a2dGeoX] = oFileMsg.latlons() except BaseException: # Get data dimension(s) if oFileMsg.has_key('Ni') and oFileMsg.has_key('Nj'): iRows = oFileMsg.Ni iCols = oFileMsg.Nj else: [iCols, iRows] = oFileMsg.values.shape # Get lats and lons using key(s) a2dGeoX = np.reshape(oFileMsg['longitudes'], [iCols, iRows]) a2dGeoY = np.reshape(oFileMsg['latitudes'], [iCols, iRows]) except BaseException: # Exit status with warning Exc.getExc(' -----> WARNING: in getVarAttr_LAMI_2i an exception occurred (lib_data_io_grib)', 2, 1) return a2dGeoX, a2dGeoY
def getTimeNow(sTimeNow='', sTimeRefType='local'): if sTimeNow is None: sTimeNow = '' if not sTimeNow == '': [sTimeNow, sTimeFormat] = correctTimeLength(sTimeNow) elif sTimeNow == '': if sTimeRefType == 'local': sTimeNow = getTimeLocal() elif sTimeRefType == 'GMT': sTimeNow = getTimeGMT() else: Exc.getExc( ' =====> WARNING: sTimeTypeRef is not defined correctly! sTimeNow initialized as local time!', 2, 1) sTimeNow = getTimeLocal() [sTimeNow, sTimeFormat] = correctTimeLength(sTimeNow) else: sTimeFormat = None Exc.getExc( ' =====> ERROR: sTimeNow format is unknown. Please check your time string!', 1, 1) return sTimeNow, sTimeFormat
def correctTimeLength(sTime): if len(sTime) == 12: sTimeUpd = sTime elif len(sTime) >= 8 and len(sTime) < 12: iTimeLength = len(sTime) iTimeLessDigits = 12 - iTimeLength sTimeLessFormat = '0' * iTimeLessDigits sTimeUpd = sTime + sTimeLessFormat elif len(sTime) > 12: Exc.getExc( ' =====> ERROR: sTime has not allowed length (greater then 12 char). sTime cannot defined!', 1, 1) elif len(sTime) < 8: Exc.getExc( ' =====> ERROR: sTime has not allowed length (less then 8 char). sTime cannot defined!', 1, 1) sTimeUpdFormat = defineTimeFormat(sTimeUpd) # Check time format definition checkTimeFormat(sTimeUpd, sTimeUpdFormat) return sTimeUpd, sTimeUpdFormat
def checkTimeFormat(sTime, sTimeFormat=sTimeFormat_Default): try: datetime.datetime.strptime(sTime, sTimeFormat) except BaseException: Exc.getExc( ' =====> ERROR: sTime has not correct format. sTime cannot defined!', 1, 1)
def importFileDict(sFileName, DataName='Data'): if exists(sFileName): try: if version_info >= (3, 0): oFileData = SourceFileLoader(DataName, sFileName).load_module() elif version_info < (3, 0): oFileData = load_source(DataName, '', open(sFileName)) oFileDict = {} for sFileVar in vars(oFileData): if not sFileVar.startswith('__'): oFileValue = getattr(oFileData, sFileVar) oFileDict[sFileVar] = oFileValue else: pass bFileData = True except BaseException: Exc.getExc(' =====> WARNING: read file ' + sFileName + ' FAILED!', 2, 1) oFileDict = None bFileData = False else: Exc.getExc(' =====> WARNING: file ' + sFileName + ' NOT FOUND!', 2, 1) oFileDict = None bFileData = False return oFileDict, bFileData
def findTimeClosest(sTime_Ref, oTime_HH=None): # Create datetime object oTime_Ref = datetime.datetime.strptime(sTime_Ref, sTimeFormat_Default) # Closest time using hours if oTime_HH is not None: a1oTime_Delta = [] for sTime_HH_Step in oTime_HH: oTime_Step = oTime_Ref.replace(hour=int(sTime_HH_Step)) oTime_Delta = oTime_Ref - oTime_Step if oTime_Delta > datetime.timedelta(seconds=1): a1oTime_Delta.append(oTime_Delta) oTime_Delta_Min = min(a1oTime_Delta) iTime_Delta_Index = a1oTime_Delta.index(oTime_Delta_Min) sTime_HH_Select = oTime_HH[iTime_Delta_Index] oTime_Select = oTime_Ref.replace(hour=int(sTime_HH_Select)) sTime_Select = oTime_Select.strftime(sTimeFormat_Default) return sTime_Select else: Exc.getExc( ' =====> WARNING: some arguments to find closest time are not defined! Check your data!', 2, 1) return sTime_Ref
def streamProcess(sOut=None, sErr=None): if sOut is None and sErr is None: return True else: Exc.getExc( ' =====> WARNING: error(s) occurred during process execution!', 2, 1) return False
def getVar3D_LAMI_2i(oFileHandle, oVarComp=None): # Check method oVarData = None try: # List conversion for string arguments of variable component(s) if isinstance(oVarComp, str) and (oVarComp is not None): oVarComp = [oVarComp] if oVarComp is None: oVarComp = [None] # Iterate over variable(s) component(s) oVarData = {} for sVarComp in oVarComp: if sVarComp is None: oFileMsg = oFileHandle() else: try: oFileMsg = oFileHandle.select(nameECMF=sVarComp) except BaseException: Exc.getExc( ' -----> WARNING: selected variable [' + sVarComp + '] is not in grib message (lib_data_io_grib)', 2, 1) oFileMsg = None # Iterate over message(s) for iVarID, oVarMsg in enumerate(oFileMsg): # Get variable name sVarName = oVarMsg.nameECMF if sVarName not in oVarData: oVarData[sVarName] = None a2dVarData_Raw = oVarMsg.values if ma.is_masked(a2dVarData_Raw): a2dVarData = a2dVarData_Raw.data else: a2dVarData = a2dVarData_Raw if oVarData[sVarName] is None: a3dVarData = np.reshape(a2dVarData, [a2dVarData.shape[0], a2dVarData.shape[1], 1]) else: a3dVarData = oVarData[sVarName] a3dVarData = np.dstack((a3dVarData, a2dVarData)) oVarData[sVarName] = a3dVarData except BaseException: # Exit status with warning Exc.getExc(' -----> WARNING: in getVar3D_LAMI_2i an exception occurred (lib_data_io_grib)', 2, 1) return oVarData
def get1DVar(oFileData, sVarName): try: a1dVarName_OUT = oFileData.variables[sVarName][:] except RuntimeError: a1dVarName_OUT = None Exc.getExc( ' =====> ERROR: in reading variable 1d. Check variable name! (lib_data_io_netcdf)', 1, 1) return a1dVarName_OUT
def checkZip(sFileName): try: oFileTest = bz2.BZ2File(sFileName).read() bFileCheck = True except BaseException: Exc.getExc( ' -----> WARNING: errors occurred in checking file integrity (lib_data_zip_bz2)', 2, 1) bFileCheck = False return bFileCheck
def unzipFile(oFile_IN=None, oFile_OUT=None): if oFile_IN and oFile_OUT: try: oDecompressData = oFile_IN.read() oFile_OUT.write(oDecompressData) except BaseException: Exc.getExc( ' -----> WARNING: errors occurred in unzipping file (lib_data_zip_bz2)', 2, 1) else: Exc.getExc(' -----> ERROR: in unzip file (lib_data_zip_bz2)', 1, 1)
def openFile(sFileName, sFileMode): import h5py try: # Open file oFile = h5py.File(sFileName, sFileMode) return oFile except IOError as oError: Exc.getExc( ' -----> ERROR: in open file (lib_data_io_hdf5)' + ' [' + str(oError) + ']', 1, 1)
def openFile(sFileName): # Check method try: # Open file oFile = gdal.Open(sFileName) return oFile except IOError as oError: Exc.getExc( ' -----> ERROR: in open file (lib_data_io_hdf4)' + ' [' + str(oError) + ']', 1, 1)
def openFile(sFileName, sFileMode): try: if 'r' in sFileMode: oFile = open(sFileName, sFileMode + 'b') elif 'w' in sFileMode: oFile = open(sFileName, sFileMode + 'b') return oFile except IOError as oError: Exc.getExc( ' -----> ERROR: in open file (lib_data_io_pickle)' + ' [' + str(oError) + ']', 1, 1)
def openFile(sFileName, sFileMode='r'): # Open file try: if sFileMode == 'r': oFile = pygrib.open(sFileName) return oFile else: Exc.getExc(' -----> WARNING: open file in write/append mode not available (lib_data_io_grib)', 2, 1) return None except IOError as oError: Exc.getExc(' -----> ERROR: in open file (lib_data_io_grib)' + ' [' + str(oError) + ']', 1, 1)
def getTimeStep(sTimeIN=None, iTimeDelta=3600, iTimeStep=0): if sTimeIN: sTimeFormat = defineTimeFormat(sTimeIN) oTimeIN = datetime.datetime.strptime(sTimeIN, sTimeFormat) oTimeOUT = oTimeIN + datetime.timedelta(seconds=iTimeStep * iTimeDelta) sTimeOUT = oTimeOUT.strftime(sTimeFormat) else: sTimeOUT = None sTimeFormat = None Exc.getExc( " =====> ERROR: sTimeIN not defined! sTimeOUT cannot calculated!", 1, 1) return sTimeOUT, sTimeFormat
def convertTimeLOCxGMT(sTime_IN=None, iTimeDiff=0): if sTime_IN: sTimeFormat_IN = defineTimeFormat(sTime_IN) sTimeFormat_OUT = sTimeFormat_IN oTime_IN = datetime.datetime.strptime(sTime_IN, sTimeFormat_IN) oTime_IN = oTime_IN.replace(minute=0, second=0, microsecond=0) oTime_OUT = oTime_IN + datetime.timedelta(seconds=iTimeDiff * 3600) sTime_OUT = oTime_OUT.strftime(sTimeFormat_OUT) else: Exc.getExc( " =====> ERROR: sTime_IN not defined! sTime_OUT cannot calculated!", 1, 1) return sTime_OUT
def getTimeFrom(sTimeTo=None, iTimeDelta=3600, iTimeStep=0): if sTimeTo: sTimeFormat = defineTimeFormat(sTimeTo) oTimeTo = datetime.datetime.strptime(sTimeTo, sTimeFormat) oTimeFrom = oTimeTo - datetime.timedelta(seconds=iTimeStep * iTimeDelta) sTimeFrom = oTimeFrom.strftime(sTimeFormat) else: sTimeFrom = None sTimeFormat = None Exc.getExc( " =====> ERROR: sTimeTo not defined! sTimeFrom cannot calculated!", 1, 1) return sTimeFrom, sTimeFormat
def getFileMessage(oFile): try: iFM = oFile.messages if iFM == 0: a1iFM = None Exc.getExc(' -----> WARNING: no message in GRIB file! Check your input file!', 2, 1) else: a1iFM = np.linspace(1, iFM, iFM, endpoint=True) except BaseException: a1iFM = None Exc.getExc(' -----> WARNING: no message in GRIB file! Check your input file!', 2, 1) return a1iFM
def checkProcess(sCLine=None, sCPath=None, iTimeError=0.001, iTimeWait=1, iTimePool=0.1): # Info checking starting oLogStream.info(' -----> Process checking: ' + sCLine + ' ... ') # Checking command before execution (to find errors) with a given time os.chdir(sCPath) oProcess = subprocess.Popen(sCLine, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) oProcess.poll() # Check process to control library or memory error(s) while oProcess.returncode is None and iTimeWait > 0: time.sleep(iTimePool) iTimeWait -= iTimePool oProcess.poll() # Condition to define if process checking is fine or not if iTimeWait <= 0: # process request stops for time out # Checking process for run until time-out iPID = oProcess.pid + 1 # There is a difference between python-pid and htop-pid (+1) os.kill(iPID, signal.SIGTERM) # Information process for run until time-out oLogStream.info(' -----> Process checking: ' + sCLine + ' ... OK [run time-out]') elif iTimeWait < iTimeError: # process request stops for errors # Checking stops for error(s) oFrameInfo = getframeinfo(currentframe()) # Information about stop for error(s) oLogStream.info(' -----> Process checking: ' + sCLine + ' ... FAILED!') Exc.getExc( ' =====> ERROR: process checking failed for library or memory errors!', 1, 1, oFrameInfo.filename, oFrameInfo.lineno) else: # Information process for run closing oLogStream.info(' -----> Process checking: ' + sCLine + ' ... OK [run ending]')
def getVar3D_ALL(oFile, sVarComp=None): # Check method try: bVarWarn = False oVarData = {} for oVar in oFile: # Get variable name sVarName = oVar.name #if not isinstance(sVarName, str): # sVarName = sVarName.encode('UTF-8') #sVarName = sVarName.replace(' ', '_').lower() if (sVarComp == sVarName) or (sVarComp is None): if sVarName not in oVarData: oVarData[sVarName] = None a2dVarData_Raw = oVar.values if ma.is_masked(a2dVarData_Raw): a2dVarData = a2dVarData_Raw.data else: a2dVarData = a2dVarData_Raw if oVarData[sVarName] is None: a3dVarData = np.reshape(a2dVarData, [a2dVarData.shape[0], a2dVarData.shape[1], 1]) else: a3dVarData = oVarData[sVarName] a3dVarData = np.dstack((a3dVarData, a2dVarData)) oVarData[sVarName] = a3dVarData else: # Exit status with warning if bVarWarn is False: Exc.getExc( ' -----> WARNING: selected variable [' + sVarComp + '] is not in grib message (lib_data_io_grib)', 2, 1) bVarWarn = True return oVarData except BaseException: # Exit status with warning Exc.getExc(' -----> WARNING: in getting 3d variable function (lib_data_io_grib)', 2, 1)
def get2DVar(oFileData, sVarName, bSetAutoMask=True): if not bSetAutoMask: Exc.getExc( ' =====> WARNING: auto_mask is set to false! (lib_data_io_netcdf)', 2, 1) oFileData.set_auto_mask(False) try: a2dVarName_IN = oFileData.variables[sVarName][:] a2dVarName_OUT = np.transpose(np.rot90(a2dVarName_IN, -1)) except RuntimeError: a2dVarName_OUT = None Exc.getExc( ' =====> ERROR: in reading variable 2d. Check variable name! (lib_data_io_netcdf)', 1, 1) return a2dVarName_OUT
def openFile(sFileName, sFileMode, sFileFormat='NETCDF4'): try: from netCDF4 import Dataset as Dataset except BaseException: from scipy.io.netcdf import netcdf_file as Dataset Exc.getExc( ' =====> WARNING: NetCDF module import from scipy (usually imported from netcdf-python)!', 2, 1) try: # NetCDF type: NETCDF3_CLASSIC , NETCDF3_64BIT , NETCDF4_CLASSIC , NETCDF4 oFile = Dataset(sFileName, sFileMode, format=sFileFormat) return oFile except IOError as oError: Exc.getExc( ' =====> ERROR: in open file (lib_data_io_netcdf)' + ' [' + str(oError) + ']', 1, 1)
def getVarGeo(oFile, iIndex=1): # Get geographical information try: # Get geographical data [a2dGeoY, a2dGeoX] = oFile[iIndex].latlons() # Check geographical orientation [a2dVarGeoX, a2dVarGeoY, bVarCorrOrient] = checkVarOrient(a2dGeoX, a2dGeoY) # Store geographical information a1oVarGeo = {'Longitude': a2dGeoX, 'Latitude': a2dGeoY, 'CorrOrient': bVarCorrOrient} except BaseException: Exc.getExc(' -----> WARNING: error in extracting file geographical data! Check input file!', 2, 1) a1oVarGeo = None # Return variable(s) return a1oVarGeo
def openFile(sFileName, sFileMode=None, sFileDelimiter=','): # Method to open an object to store features of csv dataset try: oFile = ObjCSV() if exists(sFileName): oFile.bFileData = True else: oFile.bFileData = False oFile.sFileMode = sFileMode oFile.sFileName = sFileName oFile.sFileDelimiter = sFileDelimiter return oFile except IOError as oError: Exc.getExc( ' -----> ERROR: in open file (lib_data_io_csv)' + ' [' + str(oError) + ']', 1, 1)
def openZip(sFileName_IN, sFileName_OUT, sZipMode): # Check method try: # Open file oFile_IN = None oFile_OUT = None if sZipMode == 'z': # zip mode oFile_IN = open(sFileName_IN, 'rb') oFile_OUT = bz2.BZ2File(sFileName_OUT, 'wb') elif sZipMode == 'u': # unzip mode oFile_IN = bz2.BZ2File(sFileName_IN) oFile_OUT = open(sFileName_OUT, "wb") # Pass file handle(s) return oFile_IN, oFile_OUT except IOError as oError: Exc.getExc( ' -----> ERROR: in open file (lib_data_zip_bz2)' + ' [' + str(oError) + ']', 1, 1)
def getVars(oFile, sDsetValueKey='values', sDsetAttrKey='parameters'): oFileDict = {} for sDataSet in oFile: oDataSet = oFile[sDataSet] oFileDict[sDataSet] = {} try: oFileDict[sDataSet][sDsetValueKey] = {} oFileDict[sDataSet][sDsetAttrKey] = {} oFileDict[sDataSet][sDsetValueKey] = oDataSet.value for oItem in oDataSet.attrs.keys(): oValue = oDataSet.attrs[oItem] oFileDict[sDataSet][sDsetAttrKey][oItem] = {} oFileDict[sDataSet][sDsetAttrKey][oItem] = oValue except BaseException: Exc.getExc( ' -----> Warning: impossible to get data values or attributes! Check your file! (lib_data_io_hdf5)', 2, 1) return oFileDict
def getVar2D_ALL(oFile, iIndex=None): # Check method try: if iIndex: # Read variable using Data index oVar = oFile[iIndex] else: oVar = oFile a2dVarData_Raw = oVar.values if ma.is_masked(a2dVarData_Raw): a2dVarData = a2dVarData_Raw.data else: a2dVarData = a2dVarData_Raw return a2dVarData except BaseException: # Exit status with error Exc.getExc(' -----> WARNING: in getting 2d variable function (lib_data_io_grib)', 2, 1)
def getFileAttr(oFile, iIndex=None, oVarKeyNA=None): # Default argument if oVarKeyNA is None: oVarKeyNA = ['distinctLatitudes', 'distinctLongitudes', 'values', 'latLonValues', 'latitudes', 'longitudes'] if iIndex is None: oVarHandle = oFile else: oVarHandle = oFile[iIndex] # Attributes oVarAttrs = {} for sVarKey in oVarHandle.keys(): sVarKey = sVarKey.strip().encode('UTF-8') sVarKey = sVarKey.decode('utf-8') if sVarKey not in oVarKeyNA: try: if sVarKey == 'validDate': oVarValue = oVarHandle.validDate elif sVarKey == 'analDate': oVarValue = oVarHandle.analDate else: oVarValue = oVarHandle[sVarKey] oVarAttrs.update({sVarKey: oVarValue}) except BaseException: Exc.getExc(' -----> WARNING: file key ' + sVarKey + ' not correctly retrieved Check input file!', 2, 1) else: pass return oVarAttrs