def fileMapping(oVarSource, oVarMapping, oVarTags): oVarFile = {} oVarBaseName = {} for sVarMapping_TAG, oVarMapping_NAMES in oVarMapping.items(): for sVarMapping_NAME in oVarMapping_NAMES: sVarMapping_FILE_RAW = oVarSource[sVarMapping_NAME] sVarMapping_FILE_FILLED = defineString(sVarMapping_FILE_RAW, oVarTags) oVarFile[sVarMapping_NAME] = sVarMapping_FILE_FILLED oVarBaseName[sVarMapping_NAME] = sVarMapping_FILE_RAW return oVarFile, oVarBaseName
def setDatasetTime_Finder(oVarRUN_TIMESTEP, sVarSRC_TYPEDATA, sVarANC_TYPEDATA, sVarSRC_NAME, sVarANC_NAME, oVarFILE_PATHREF, oVarFILE_PATHGENERIC, oVarFILE_MEMORY, oVarTIME_MEMORY, oVarSRC_ENSEMBLE=None, iVarSRC_FREQ='D', iVarSRC_PERIOD=3, iVarANC_FREQ='D', iVarANC_PERIOD=3, sVarSCAN_Freq='H'): # Compute time-range for RUN period oVarRUN_TIME = pd.date_range(end=oVarRUN_TIMESTEP, periods=iVarSRC_PERIOD, freq=iVarSRC_FREQ) oVarRUN_TIME = pd.date_range(start=oVarRUN_TIME[0], end=oVarRUN_TIME[-1], freq=sVarSCAN_Freq) oVarRUN_TIME = oVarRUN_TIME.sort_values(return_indexer=False, ascending=False) if sVarANC_NAME is not None: oDFrame_TYPE = [sVarSRC_TYPEDATA, sVarANC_TYPEDATA] oDFrame_COLS = [sVarSRC_NAME, sVarANC_NAME] else: oDFrame_TYPE = [sVarSRC_TYPEDATA] oDFrame_COLS = [sVarSRC_NAME] oDFrame_DATA = pd.DataFrame(index=oVarRUN_TIME, columns=oDFrame_COLS) oDFrame_DATA.fillna(False) # Get generic path(s) of SOURCE and ANCILLARY sVarSRC_PATHGENERIC = oVarFILE_PATHGENERIC[sVarSRC_NAME] sVarSRC_PATHBASE, sVarSRC_FILEBASE = os.path.split(sVarSRC_PATHGENERIC) if sVarANC_NAME is not None: sVarANC_PATHGENERIC = oVarFILE_PATHGENERIC[sVarANC_NAME] sVarANC_PATHBASE, sVarANC_FILEBASE = os.path.split(sVarANC_PATHGENERIC) # Iterate over time-range oVarSRC_TIME = {} oVarANC_TIME = {} for oVarRUN_TIMECHECK in oVarRUN_TIME: oVarSRC_TIMECHECK = oVarRUN_TIMECHECK oListSRC_TAGS = setDatasetTags_Finder(oVarSRC_TIMECHECK, oVarSRC_ENSEMBLE) oVarSRC_TIME[oVarRUN_TIMECHECK] = {} for oElemSRC_TAGS in oListSRC_TAGS: sVarSRC_PATHCHECK = defineString(sVarSRC_PATHGENERIC, oElemSRC_TAGS) oVarSRC_FILECHECK = glob.glob(sVarSRC_PATHCHECK) if oVarSRC_FILECHECK: if 'data' not in oVarSRC_TIME[oVarRUN_TIMECHECK]: oVarSRC_TIME[oVarRUN_TIMECHECK]['time'] = oVarSRC_TIMECHECK oVarSRC_TIME[oVarRUN_TIMECHECK]['data'] = oVarSRC_FILECHECK else: oVarSRC_FILESAVE = oVarSRC_TIME[oVarRUN_TIMECHECK]['data'] oVarSRC_FILESAVE.append(oVarSRC_FILECHECK[0]) oVarSRC_TIME[oVarRUN_TIMECHECK]['data'] = oVarSRC_FILESAVE if not oVarSRC_TIME[oVarRUN_TIMECHECK]: oVarSRC_TIME[oVarRUN_TIMECHECK] = None for oVarRUN_TIME, oVarSRC_DATA in oVarSRC_TIME.items(): if oVarSRC_DATA is not None: oVarSRC_TIMESTEP = oVarSRC_DATA['time'] oDFrame_DATA.loc[oVarRUN_TIME, sVarSRC_NAME] = oVarSRC_TIMESTEP if sVarANC_NAME is not None: oVarANC_TIMEHISTORY = [] if oVarSRC_TIME: for oVarRUN_TIMECHECK, oVarRUN_DATA in oVarSRC_TIME.items(): oVarANC_TIMERANGE = pd.date_range(start=oVarRUN_TIMECHECK, closed=None, periods=iVarANC_PERIOD, freq=iVarANC_FREQ) oVarANC_TIMESCAN = pd.date_range(start=oVarANC_TIMERANGE[0], end=oVarANC_TIMERANGE[-1], freq=sVarSCAN_Freq) oVarANC_TIME[oVarRUN_TIMECHECK] = {} #for oVarANC_TIMECHECK in oVarANC_TIMERANGE: for oVarANC_TIMECHECK in oVarANC_TIMESCAN: if oVarANC_TIMECHECK <= oVarRUN_TIMESTEP: if oVarANC_TIMECHECK not in oVarANC_TIMEHISTORY: oVarANC_TIMEHISTORY.append(oVarANC_TIMECHECK) oListANC_TAGS = setDatasetTags_Finder( oVarANC_TIMECHECK, oVarSRC_ENSEMBLE) for oElemANC_TAGS in oListANC_TAGS: sVarANC_PATHCHECK = defineString( sVarANC_PATHGENERIC, oElemANC_TAGS) oVarANC_FILECHECK = glob.glob( sVarANC_PATHCHECK) if oVarANC_FILECHECK: if 'data' not in oVarANC_TIME[ oVarRUN_TIMECHECK]: oVarANC_TIME[oVarRUN_TIMECHECK][ 'time'] = oVarANC_TIMECHECK oVarANC_TIME[oVarRUN_TIMECHECK][ 'data'] = oVarANC_FILECHECK else: oVarANC_FILESAVE = oVarANC_TIME[ oVarRUN_TIMECHECK]['data'] oVarANC_FILESAVE.append( oVarANC_FILECHECK[0]) oVarANC_TIME[oVarRUN_TIMECHECK][ 'data'] = oVarANC_FILESAVE if not oVarANC_TIME[oVarRUN_TIMECHECK]: oVarANC_TIME[oVarRUN_TIMECHECK] = None for oVarRUN_TIME, oVarANC_DATA in oVarANC_TIME.items(): if oVarANC_DATA is not None: oVarANC_TIMESTEP = oVarANC_DATA['time'] oDFrame_DATA.loc[oVarRUN_TIME, sVarANC_NAME] = oVarANC_TIMESTEP # Iteration(s) to find datetime(s) of SRC and ANC dataset(s) oDFrame_DATA_REVERSE = oDFrame_DATA[::-1] oDict_DATA_SEARCH = None oDFrame_DATA_SEARCH = pd.DataFrame(columns=oDFrame_COLS) for oDFrame_ROW in oDFrame_DATA_REVERSE.iterrows(): oVarTIME_ROW = pd.DatetimeIndex([oDFrame_ROW[0]]) oVarDATA_ROW = list(oDFrame_ROW[1].values) if oDict_DATA_SEARCH is None: oDict_DATA_SEARCH = dict.fromkeys(oDFrame_COLS, None) for sDFrame_COLS, sDFrame_TYPE, oDFrame_VALUE in zip( oDFrame_COLS, oDFrame_TYPE, oVarDATA_ROW): if isinstance(oDFrame_VALUE, (datetime.datetime, np.datetime64, datetime.date)): if not pd.isnull(oDFrame_VALUE): oDict_DATA_SEARCH[sDFrame_COLS] = oDFrame_VALUE if sDFrame_TYPE == 'result': oDict_DATA_VALUES = list(oDict_DATA_SEARCH.values()) for sElem_DATA_COLS, oElem_DATA_VALUES in zip( oDFrame_COLS, oDict_DATA_VALUES): if oElem_DATA_VALUES is not None: if not pd.isnull(oDFrame_VALUE): if oDFrame_VALUE < oElem_DATA_VALUES: oDict_DATA_SEARCH[sElem_DATA_COLS] = None if sDFrame_TYPE == 'observed': oDict_DATA_VALUES = list(oDict_DATA_SEARCH.values()) for sElem_DATA_COLS, oElem_DATA_VALUES in zip( oDFrame_COLS, oDict_DATA_VALUES): if oElem_DATA_VALUES is not None: if not pd.isnull(oDFrame_VALUE): if oDFrame_VALUE > oElem_DATA_VALUES: oDict_DATA_SEARCH[sElem_DATA_COLS] = None if sDFrame_TYPE == 'forecast': oDict_DATA_VALUES = list(oDict_DATA_SEARCH.values()) for sElem_DATA_COLS, oElem_DATA_VALUES in zip( oDFrame_COLS, oDict_DATA_VALUES): if oElem_DATA_VALUES is not None: if not pd.isnull(oDFrame_VALUE): if oDFrame_VALUE > oElem_DATA_VALUES: oDict_DATA_SEARCH[sElem_DATA_COLS] = None if None not in oDict_DATA_SEARCH.values(): oDFrame_DATA_SEARCH = oDFrame_DATA_SEARCH.append( pd.DataFrame(oDict_DATA_SEARCH, index=oVarTIME_ROW)) oDict_DATA_SEARCH = None # break # Append value for last part to get also not completed dictionary if oDict_DATA_SEARCH is not None: if any(oElem is not None for oElem in oDict_DATA_SEARCH.values()): oDFrame_DATA_SEARCH = oDFrame_DATA_SEARCH.append( pd.DataFrame(oDict_DATA_SEARCH, index=oVarTIME_ROW)) # Delete rows with all nan(s) oDFrame_DATA_ALL = oDFrame_DATA_SEARCH.dropna(axis=0, how='all') oDFrame_DATA_FINITE = oDFrame_DATA_SEARCH.dropna(axis=0, how='any') if not oDFrame_DATA_FINITE.empty: oDFrame_DATA_SELECT = oDFrame_DATA_FINITE else: Exc.getExc( ' =====> WARNING: source and/or ancillary columns is/are null!', 2, 1) oDFrame_DATA_SELECT = oDFrame_DATA_ALL # Initialize time dictionary (if none is actual value) if oVarTIME_MEMORY is None: oVarTIME_MEMORY = {} # Iterate over time select for oDFrame_ROW_SELECT in oDFrame_DATA_SELECT.iterrows(): # Iterate over columns of SRC and ANC name(s) for sDFrame_COLS, sDFrame_TYPE in zip(oDFrame_COLS, oDFrame_TYPE): # Check data availability if oDFrame_ROW_SELECT: # Get selected time oVarTIME_ROW_SELECT = oDFrame_ROW_SELECT[1][sDFrame_COLS] if sDFrame_COLS in list(oVarTIME_MEMORY.keys()): oVarFILE_TIMESAVED = oVarTIME_MEMORY[sDFrame_COLS] # oVarFILE_TIMEACTUAL = pd.Timestamp(oDFrame_ROW_SELECT[sDFrame_COLS].values[0]) oVarFILE_TIMEACTUAL = oVarTIME_ROW_SELECT if oVarFILE_TIMEACTUAL > oVarFILE_TIMESAVED: oVarTIME_MEMORY[sDFrame_COLS] = None oVarFILE_MEMORY[sDFrame_COLS] = False # Check to update path(s) only one time if not oVarFILE_MEMORY[sDFrame_COLS]: sVarFILE_PATHSELECT = oVarFILE_PATHGENERIC[sDFrame_COLS] # oVarFILE_TIMESELECT = pd.Timestamp(oDFrame_DATA_SELECT[sDFrame_COLS].values[0]) oVarFILE_TIMESELECT = oVarTIME_ROW_SELECT oListFILE_TAGS = setDatasetTags_Finder( oVarFILE_TIMESELECT)[0] sVarFILE_PATHBASE, sVarFILE_FILEBASE = os.path.split( sVarFILE_PATHSELECT) if sDFrame_TYPE == 'observed': sVarFILE_PATHREF = join(sVarFILE_PATHBASE, sVarFILE_FILEBASE) elif (sDFrame_TYPE == 'forecast') or (sDFrame_TYPE == 'result'): sVarFILE_PATHDEF = defineString( sVarFILE_PATHBASE, oListFILE_TAGS) sVarFILE_PATHREF = join(sVarFILE_PATHDEF, sVarFILE_FILEBASE) oVarTIME_MEMORY[sDFrame_COLS] = oVarFILE_TIMESELECT oVarFILE_PATHREF[sDFrame_COLS] = sVarFILE_PATHREF oVarFILE_MEMORY[sDFrame_COLS] = True else: Exc.getExc( ' =====> WARNING: dataframe for ' + sDFrame_COLS + ' variable is null!', 2, 1) oVarTIME_MEMORY[sDFrame_COLS] = oVarRUN_TIMESTEP oVarFILE_MEMORY[sDFrame_COLS] = True return oVarTIME_MEMORY, oVarFILE_PATHREF, oVarFILE_MEMORY
def main(): # ------------------------------------------------------------------------------------- # Version and algorithm information sProgramVersion = '1.2.0' sProjectName = 'HAT' sAlgType = 'Analyzer_Datasets' sAlgName = 'HMC NearRealTime' # Time algorithm information dStartTime = time.time() # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Get script argument(s) [sScriptName, sFileSetting, sTimeArg] = GetArgs() # Set algorithm configuration oDrv_Data_Settings = DataAlgorithm(sFileSetting) [oData_Settings, oData_Path, oData_Flags, oData_Mapping, oData_ColorMap] = oDrv_Data_Settings.getDataSettings() # Set logging file oLogStream = setLoggingFile( oData_Path['log'], bLoggerHistory=oData_Settings['data']['log']['history']) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Start Program oLogStream.info('[' + sProjectName + ' ' + sAlgType + ' - ' + sAlgName + ' (Version ' + sProgramVersion + ')]') oLogStream.info('[' + sProjectName + '] Start Program ... ') # Clean static ancillary file(s) and folder(s) DataAnalysisCleaner( flag=[oData_Flags['cleaning_static_data']], file=[oData_Path['section_ancillary'], oData_Path['land_ancillary']]).cleanDataAnalysis() # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Get algorithm time oLogStream.info(' --> Set algorithm time ... ') oDrv_Time = DataTime( sTimeArg, sTimeNow=oData_Settings['time']['time_now'], iTimePeriodPast=oData_Settings['time']['time_period'], sTimeFrequency=oData_Settings['time']['time_frequency'], sTimeETA=oData_Settings['time']['time_eta']) oData_TimeRun = oDrv_Time.getDataTime(bTimeReverse=True, bTimeRestart=False) oLogStream.info(' --> Set algorithm time ... DONE') # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Get geographical data oLogStream.info(' --> Set geographical data ... ') oDrv_Data_Geo = DataGeo( file_terrain=oData_Path['land_terrain'], file_channel_network=oData_Path['land_channel_network'], file_ancillary=oData_Path['land_ancillary'], vars=['Terrain', 'Channel_Network']) oData_Geo = oDrv_Data_Geo.getDataGeo() oLogStream.info(' --> Set geographical data ... DONE') # Get sections data oLogStream.info(' --> Set section data ... ') oDrv_Data_Section = DataPoint( file_shp=oData_Path['section_ref'], file_mask=oData_Path['section_mask'], file_terrain=oData_Path['section_dem'], file_ctime=oData_Path['section_ctime'], file_ancillary=oData_Path['section_ancillary'], fields=None, ) oData_Section = oDrv_Data_Section.getDataPoint(oData_Geo) oLogStream.info(' --> Set section data ... DONE') # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Iterate over time steps for oTimeRun_STEP in oData_TimeRun['oTimeSteps']: # ------------------------------------------------------------------------------------- # Get data time oLogStream.info(' --> Get dynamic time ... ') oDrv_DataTime_Dynamic = DataAnalysisTime(time=oTimeRun_STEP, settings=oData_Settings) oData_TimePeriod = oDrv_DataTime_Dynamic.computeDataTime(oData_Geo) oLogStream.info(' --> Get dynamic time ... DONE') # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Set data tags /$yyyy/$mm/$dd/ oDrv_Data_Tags = DataTags({ '$yyyy': str(oTimeRun_STEP.year).zfill(4), '$mm': str(oTimeRun_STEP.month).zfill(2), '$dd': str(oTimeRun_STEP.day).zfill(2), '$HH': str(oTimeRun_STEP.hour).zfill(2), '$MM': str(oTimeRun_STEP.minute).zfill(2), '$RUNTIME': str(oTimeRun_STEP.year).zfill(4) + '/' + str(oTimeRun_STEP.month).zfill(2) + '/' + str(oTimeRun_STEP.day).zfill(2) + '/' + str(oTimeRun_STEP.hour).zfill(2), '$RUNDOMAIN': oData_Settings['algorithm']['ancillary']['run_domain'], '$RUNNAME': oData_Settings['algorithm']['ancillary']['run_name'] }) oData_Tags = oDrv_Data_Tags.setDataTags() # Info time oLogStream.info(' --> Set time step: ' + str(oTimeRun_STEP)) # Clean buffer and product file(s) and folder(s) DataAnalysisCleaner( flag=[ oData_Flags['cleaning_dynamic_data_source'], oData_Flags['cleaning_dynamic_data_outcome'], ], file=[[ defineString(oData_Path['file_buffer_ts'], oData_Tags), defineString(oData_Path['file_buffer_gridded'], oData_Tags), defineString(oData_Path['file_buffer_info'], oData_Tags) ], [ defineString(oData_Path['file_buffer_registry'], oData_Tags), [ defineString(oData_Path[sListItem], oData_Tags) for sListItem in list( oData_Mapping['file_dewetra_data_ts']) ], [ defineString(oData_Path[sListItem], oData_Tags) for sListItem in list( oData_Mapping['file_dewetra_data_gridded']) ], [ defineString(oData_Path[sListItem], oData_Tags) for sListItem in list( oData_Mapping['file_hydrapp_graph_ts']) ], [ defineString(oData_Path[sListItem], oData_Tags) for sListItem in list( oData_Mapping['file_hydrapp_graph_gridded']) ], ]]).cleanDataAnalysis() # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Get data analysis oLogStream.info(' --> Seek analysis data ... ') oDrv_DataSeeker_Dynamic = DataAnalysisSeeker( oData_Settings['algorithm']['ancillary']['run_application'], time_seek=oTimeRun_STEP, time_period=oData_TimePeriod, settings=oData_Settings, file=oData_Path, mapping=oData_Mapping, tags=oData_Tags) if not exists(defineString(oData_Path['file_buffer_info'], oData_Tags)): oData_Dynamic_Source_Info = oDrv_DataSeeker_Dynamic.seekDataAnalysis( oData_Section) savePickle( defineString(oData_Path['file_buffer_info'], oData_Tags), oData_Dynamic_Source_Info) oLogStream.info(' --> Seek dynamic data ... DONE') else: oData_Dynamic_Source_Info = restorePickle( defineString(oData_Path['file_buffer_info'], oData_Tags)) oLogStream.info( ' --> Seek dynamic data ... LOADED from workspace saved file.') # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Make data analysis oLogStream.info(' --> Make analysis data... ') oDrv_DataMaker_Dynamic = DataAnalysisMaker( oData_Settings['algorithm']['ancillary']['run_application'], time_make=oTimeRun_STEP, time_period=oData_TimePeriod, settings=oData_Settings, file=oData_Path, mapping=oData_Mapping, cmap=oData_ColorMap, tags=oData_Tags) if not exists( defineString(oData_Path['file_buffer_registry'], oData_Tags)): oData_Dynamic_Outcome_Info = oDrv_DataMaker_Dynamic.selectDataAnalysisMaker( oData_Dynamic_Source_Info, oData_Geo, oData_Section) savePickle( defineString(oData_Path['file_buffer_registry'], oData_Tags), oData_Dynamic_Outcome_Info) oLogStream.info(' --> Make analysis data ... DONE') else: oLogStream.info( ' --> Make analysis data ... SKIPPED! Data previously published.' ) oData_Dynamic_Outcome_Info = restorePickle( defineString(oData_Path['file_buffer_registry'], oData_Tags)) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Clean buffer file(s) and folder(s) DataAnalysisCleaner( flag=[ oData_Flags['cleaning_dynamic_data_source'], oData_Flags['cleaning_dynamic_data_outcome'] ], file=[[ defineString(oData_Path['file_buffer_ts'], oData_Tags), defineString(oData_Path['file_buffer_gridded'], oData_Tags), defineString(oData_Path['file_buffer_info'], oData_Tags) ], [defineString(oData_Path['file_buffer_registry'], oData_Tags)]]).cleanDataAnalysis() # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Note about script parameter(s) oLogStream.info('NOTE - Algorithm parameter(s)') oLogStream.info('Script: ' + str(sScriptName)) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # End Program dTimeElapsed = round(time.time() - dStartTime, 1) oLogStream.info('[' + sProjectName + ' ' + sAlgType + ' - ' + sAlgName + ' (Version ' + sProgramVersion + ')]') oLogStream.info('End Program - Time elapsed: ' + str(dTimeElapsed) + ' seconds') Exc.getExc('', 0, 0)
def defineDataTags(sString, oTags): return defineString(sString, oTags)
def __savePointInfo(self, oDataPoint, oDataGeo): # ------------------------------------------------------------------------------------- # Initialise section workspace oDataPoint['SEC_FILE_ANCILLARY'] = None oDataPoint['SEC_FILE_TERRAIN'] = None oDataPoint['SEC_FILE_CTIME'] = None oDataPoint['SEC_FILE_MASK'] = None # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Iterate over section(s) for iSectionID, oSectionPoint in enumerate(oDataPoint.iterrows()): # ------------------------------------------------------------------------------------- # Define section info oSectionData = oSectionPoint[1] sSectionName = oSectionData['SEC_TAG'] sSectionName = sSectionName.lower().replace(':', '') # Info start oLogStream.info(' ----> Save point info for section ' + sSectionName + ' ... ') # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Define filename(s) sFileName_MASK = defineString(self.__sFileName_MASK, {'$section': sSectionName}) sFileName_TERRAIN = defineString(self.__sFileName_TERRAIN, {'$section': sSectionName}) sFileName_CTIME = defineString(self.__sFileName_CTIME, {'$section': sSectionName}) sFileName_ANC = defineString(self.__sFileName_ANC, {'$section': sSectionName}) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Check ancillary section file availability if not isfile(sFileName_ANC): # ------------------------------------------------------------------------------------- # Get mask information if isfile(sFileName_MASK): # Open ascii grid data oFileDriver = Drv_Data_IO(sFileName_MASK).oFileWorkspace oFileData = oFileDriver.oFileLibrary.openFile( join(oFileDriver.sFilePath, oFileDriver.sFileName), 'r') # Read ascii grid data [a2iVarMask, a1oHeaderMask ] = oFileDriver.oFileLibrary.readArcGrid(oFileData) a2iVarMask = np.flipud(a2iVarMask) else: Exc.getExc( ' =====> WARNING: file mask undefined (' + sFileName_MASK + ')', 2, 1) a2iVarMask = None a1oHeaderMask = None # Get terrain information if isfile(sFileName_TERRAIN): # Open ascii grid data oFileDriver = Drv_Data_IO(sFileName_TERRAIN).oFileWorkspace oFileData = oFileDriver.oFileLibrary.openFile( join(oFileDriver.sFilePath, oFileDriver.sFileName), 'r') # Read ascii grid data [a2dVarTerrain, a1oHeaderTerrain ] = oFileDriver.oFileLibrary.readArcGrid(oFileData) a2dVarTerrain = np.flipud(a2dVarTerrain) else: Exc.getExc( ' =====> WARNING: file terrain undefined (' + sFileName_TERRAIN + ')', 2, 1) a2dVarTerrain = None a1oHeaderTerrain = None # Get corrivation time information if isfile(sFileName_CTIME): # Open ascii grid data oFileDriver = Drv_Data_IO(sFileName_CTIME).oFileWorkspace oFileData = oFileDriver.oFileLibrary.openFile( join(oFileDriver.sFilePath, oFileDriver.sFileName), 'r') # Read ascii grid data [a2dVarCorrTime, a1oHeaderCorrTime ] = oFileDriver.oFileLibrary.readArcGrid(oFileData) a2dVarCorrTime = np.flipud(a2dVarCorrTime) else: Exc.getExc( ' =====> WARNING: file corrivation_time undefined (' + sFileName_CTIME + ')', 2, 1) a2dVarCorrTime = None a1oHeaderCorrTime = None # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Check mask, terrain and corrivation time information if (a2iVarMask is not None) and (a2dVarCorrTime is not None) and (a2dVarTerrain is not None): # ------------------------------------------------------------------------------------- # Read geographical header [ iRows, iCols, dGeoXMin, dGeoYMin, dGeoXStep, dGeoYStep, dNoData ] = readGeoHeader(a1oHeaderTerrain) # Define geographical corners [dGeoXMin, dGeoXMax, dGeoYMin, dGeoYMax] = defineGeoCorner(dGeoXMin, dGeoYMin, dGeoXStep, dGeoYStep, iCols, iRows) # Define geographical grid [a2dGeoX, a2dGeoY, a1dGeoBox] = defineGeoGrid(dGeoYMin, dGeoXMin, dGeoYMax, dGeoXMax, dGeoYStep, dGeoXStep) # Compute section drainage area a2dCellArea = computeCellArea(a2dGeoY, dGeoXStep, dGeoYStep) dCellArea = np.nanmean(a2dCellArea) dDrainageArea = computeDrainageArea( a2dVarTerrain, dCellArea) # Add drainage area to attribute(s) oSectionData['DrainageArea'] = dDrainageArea # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Define encoding oGeoEncoding = { 'Mask': { '_FillValue': a1oHeaderMask['NODATA_value'] }, 'Terrain': { '_FillValue': a1oHeaderTerrain['NODATA_value'] }, 'CorrTime': { '_FillValue': a1oHeaderCorrTime['NODATA_value'] } } # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Create data array for mask oVarMask = createVar2D(a2iVarMask, oDataGeo.a2dGeoX, oDataGeo.a2dGeoY, sVarName='Mask') # Create data array for terrain oVarTerrain = createVar2D(a2dVarTerrain, oDataGeo.a2dGeoX, oDataGeo.a2dGeoY, sVarName='Terrain') # Create data array for channel network oVarCorrTime = createVar2D(a2dVarCorrTime, oDataGeo.a2dGeoX, oDataGeo.a2dGeoY, sVarName='CorrTime') # Merge data arrays in a common dataset oVarDSet = mergeVar2D( [oVarMask, oVarTerrain, oVarCorrTime]) # Add attribute to section dataset for oSectionKey, oSectionValue in oSectionData.items(): if oSectionKey != 'geometry': if oSectionValue is not None: oVarDSet.attrs[oSectionKey] = oSectionValue # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Save data in a netcdf file if not isfile(sFileName_ANC): createFolderByFile(sFileName_ANC) sVarMode = 'w' else: sVarMode = 'a' # Dump data to netcdf file writeFileNC4(sFileName_ANC, oVarDSet, oVarAttr=oGeoEncoding, sVarGroup=None, sVarMode=sVarMode, oVarEngine='h5netcdf', iVarCompression=9) # Info end oLogStream.info(' ----> Save point info for section ' + sSectionName + ' ... OK') # ------------------------------------------------------------------------------------- else: # ------------------------------------------------------------------------------------- # Info exit Exc.getExc( ' =====> WARNING: not enough information to save info section file', 2, 1) oLogStream.info(' ----> Save point info for section ' + sSectionName + ' ... FAILED') # ------------------------------------------------------------------------------------- else: # ------------------------------------------------------------------------------------- # Info skip oLogStream.info( ' ----> Save point info ... previously done. SKIPPED') # Get data processed previously to extract some information oFileHandle = readFileNC4(sFileName_ANC) if hasattr(oFileHandle, 'DrainageArea'): dDrainageArea = getattr(oFileHandle, 'DrainageArea') else: dDrainageArea = -9999.0 Exc.getExc( ' =====> WARNING: drainage area value not available! Set to -9999.0', 2, 1) # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Update data point with defined filename oDataPoint.at[iSectionID, 'OUTLET_FILE_ANCILLARY'] = sFileName_ANC oDataPoint.at[iSectionID, 'OUTLET_FILE_TERRAIN'] = sFileName_TERRAIN oDataPoint.at[iSectionID, 'OUTLET_FILE_CTIME'] = sFileName_CTIME oDataPoint.at[iSectionID, 'OUTLET_FILE_MASK'] = sFileName_MASK oDataPoint.at[iSectionID, 'OUTLET_NAME'] = sSectionName oDataPoint.at[iSectionID, 'DrainageArea'] = dDrainageArea # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Sort data point using BASIN and DrainageArea as pivot(s) oDataPoint_SORT = oDataPoint.sort_values(['BASIN', 'DrainageArea']) oDataPoint_SORT['HAT_ELEMENT'] = oDataPoint_SORT.groupby( 'BASIN')['BASIN'].transform('count') oDataPoint_SORT["HAT_ID_ROWS"] = oDataPoint_SORT['BASIN'].map({ iX: iI for iI, iX in enumerate(oDataPoint_SORT['BASIN'].unique()) }) + 1 oDataPoint_SORT["HAT_ID_COLS"] = oDataPoint_SORT.groupby( "BASIN").cumcount() + 1 # ------------------------------------------------------------------------------------- # ------------------------------------------------------------------------------------- # Return update points workspace return oDataPoint_SORT