def fileMapping(oVarSource, oVarMapping, oVarTags):
oVarFile = {}
oVarBaseName = {}
for sVarMapping_TAG, oVarMapping_NAMES in oVarMapping.items():
for sVarMapping_NAME in oVarMapping_NAMES:
sVarMapping_FILE_RAW = oVarSource[sVarMapping_NAME]
sVarMapping_FILE_FILLED = defineString(sVarMapping_FILE_RAW, oVarTags)
oVarFile[sVarMapping_NAME] = sVarMapping_FILE_FILLED
oVarBaseName[sVarMapping_NAME] = sVarMapping_FILE_RAW
return oVarFile, oVarBaseName
def setDatasetTime_Finder(oVarRUN_TIMESTEP,
sVarSRC_TYPEDATA,
sVarANC_TYPEDATA,
sVarSRC_NAME,
sVarANC_NAME,
oVarFILE_PATHREF,
oVarFILE_PATHGENERIC,
oVarFILE_MEMORY,
oVarTIME_MEMORY,
oVarSRC_ENSEMBLE=None,
iVarSRC_FREQ='D',
iVarSRC_PERIOD=3,
iVarANC_FREQ='D',
iVarANC_PERIOD=3,
sVarSCAN_Freq='H'):
# Compute time-range for RUN period
oVarRUN_TIME = pd.date_range(end=oVarRUN_TIMESTEP,
periods=iVarSRC_PERIOD,
freq=iVarSRC_FREQ)
oVarRUN_TIME = pd.date_range(start=oVarRUN_TIME[0],
end=oVarRUN_TIME[-1],
freq=sVarSCAN_Freq)
oVarRUN_TIME = oVarRUN_TIME.sort_values(return_indexer=False,
ascending=False)
if sVarANC_NAME is not None:
oDFrame_TYPE = [sVarSRC_TYPEDATA, sVarANC_TYPEDATA]
oDFrame_COLS = [sVarSRC_NAME, sVarANC_NAME]
else:
oDFrame_TYPE = [sVarSRC_TYPEDATA]
oDFrame_COLS = [sVarSRC_NAME]
oDFrame_DATA = pd.DataFrame(index=oVarRUN_TIME, columns=oDFrame_COLS)
oDFrame_DATA.fillna(False)
# Get generic path(s) of SOURCE and ANCILLARY
sVarSRC_PATHGENERIC = oVarFILE_PATHGENERIC[sVarSRC_NAME]
sVarSRC_PATHBASE, sVarSRC_FILEBASE = os.path.split(sVarSRC_PATHGENERIC)
if sVarANC_NAME is not None:
sVarANC_PATHGENERIC = oVarFILE_PATHGENERIC[sVarANC_NAME]
sVarANC_PATHBASE, sVarANC_FILEBASE = os.path.split(sVarANC_PATHGENERIC)
# Iterate over time-range
oVarSRC_TIME = {}
oVarANC_TIME = {}
for oVarRUN_TIMECHECK in oVarRUN_TIME:
oVarSRC_TIMECHECK = oVarRUN_TIMECHECK
oListSRC_TAGS = setDatasetTags_Finder(oVarSRC_TIMECHECK,
oVarSRC_ENSEMBLE)
oVarSRC_TIME[oVarRUN_TIMECHECK] = {}
for oElemSRC_TAGS in oListSRC_TAGS:
sVarSRC_PATHCHECK = defineString(sVarSRC_PATHGENERIC,
oElemSRC_TAGS)
oVarSRC_FILECHECK = glob.glob(sVarSRC_PATHCHECK)
if oVarSRC_FILECHECK:
if 'data' not in oVarSRC_TIME[oVarRUN_TIMECHECK]:
oVarSRC_TIME[oVarRUN_TIMECHECK]['time'] = oVarSRC_TIMECHECK
oVarSRC_TIME[oVarRUN_TIMECHECK]['data'] = oVarSRC_FILECHECK
else:
oVarSRC_FILESAVE = oVarSRC_TIME[oVarRUN_TIMECHECK]['data']
oVarSRC_FILESAVE.append(oVarSRC_FILECHECK[0])
oVarSRC_TIME[oVarRUN_TIMECHECK]['data'] = oVarSRC_FILESAVE
if not oVarSRC_TIME[oVarRUN_TIMECHECK]:
oVarSRC_TIME[oVarRUN_TIMECHECK] = None
for oVarRUN_TIME, oVarSRC_DATA in oVarSRC_TIME.items():
if oVarSRC_DATA is not None:
oVarSRC_TIMESTEP = oVarSRC_DATA['time']
oDFrame_DATA.loc[oVarRUN_TIME, sVarSRC_NAME] = oVarSRC_TIMESTEP
if sVarANC_NAME is not None:
oVarANC_TIMEHISTORY = []
if oVarSRC_TIME:
for oVarRUN_TIMECHECK, oVarRUN_DATA in oVarSRC_TIME.items():
oVarANC_TIMERANGE = pd.date_range(start=oVarRUN_TIMECHECK,
closed=None,
periods=iVarANC_PERIOD,
freq=iVarANC_FREQ)
oVarANC_TIMESCAN = pd.date_range(start=oVarANC_TIMERANGE[0],
end=oVarANC_TIMERANGE[-1],
freq=sVarSCAN_Freq)
oVarANC_TIME[oVarRUN_TIMECHECK] = {}
#for oVarANC_TIMECHECK in oVarANC_TIMERANGE:
for oVarANC_TIMECHECK in oVarANC_TIMESCAN:
if oVarANC_TIMECHECK <= oVarRUN_TIMESTEP:
if oVarANC_TIMECHECK not in oVarANC_TIMEHISTORY:
oVarANC_TIMEHISTORY.append(oVarANC_TIMECHECK)
oListANC_TAGS = setDatasetTags_Finder(
oVarANC_TIMECHECK, oVarSRC_ENSEMBLE)
for oElemANC_TAGS in oListANC_TAGS:
sVarANC_PATHCHECK = defineString(
sVarANC_PATHGENERIC, oElemANC_TAGS)
oVarANC_FILECHECK = glob.glob(
sVarANC_PATHCHECK)
if oVarANC_FILECHECK:
if 'data' not in oVarANC_TIME[
oVarRUN_TIMECHECK]:
oVarANC_TIME[oVarRUN_TIMECHECK][
'time'] = oVarANC_TIMECHECK
oVarANC_TIME[oVarRUN_TIMECHECK][
'data'] = oVarANC_FILECHECK
else:
oVarANC_FILESAVE = oVarANC_TIME[
oVarRUN_TIMECHECK]['data']
oVarANC_FILESAVE.append(
oVarANC_FILECHECK[0])
oVarANC_TIME[oVarRUN_TIMECHECK][
'data'] = oVarANC_FILESAVE
if not oVarANC_TIME[oVarRUN_TIMECHECK]:
oVarANC_TIME[oVarRUN_TIMECHECK] = None
for oVarRUN_TIME, oVarANC_DATA in oVarANC_TIME.items():
if oVarANC_DATA is not None:
oVarANC_TIMESTEP = oVarANC_DATA['time']
oDFrame_DATA.loc[oVarRUN_TIME,
sVarANC_NAME] = oVarANC_TIMESTEP
# Iteration(s) to find datetime(s) of SRC and ANC dataset(s)
oDFrame_DATA_REVERSE = oDFrame_DATA[::-1]
oDict_DATA_SEARCH = None
oDFrame_DATA_SEARCH = pd.DataFrame(columns=oDFrame_COLS)
for oDFrame_ROW in oDFrame_DATA_REVERSE.iterrows():
oVarTIME_ROW = pd.DatetimeIndex([oDFrame_ROW[0]])
oVarDATA_ROW = list(oDFrame_ROW[1].values)
if oDict_DATA_SEARCH is None:
oDict_DATA_SEARCH = dict.fromkeys(oDFrame_COLS, None)
for sDFrame_COLS, sDFrame_TYPE, oDFrame_VALUE in zip(
oDFrame_COLS, oDFrame_TYPE, oVarDATA_ROW):
if isinstance(oDFrame_VALUE,
(datetime.datetime, np.datetime64, datetime.date)):
if not pd.isnull(oDFrame_VALUE):
oDict_DATA_SEARCH[sDFrame_COLS] = oDFrame_VALUE
if sDFrame_TYPE == 'result':
oDict_DATA_VALUES = list(oDict_DATA_SEARCH.values())
for sElem_DATA_COLS, oElem_DATA_VALUES in zip(
oDFrame_COLS, oDict_DATA_VALUES):
if oElem_DATA_VALUES is not None:
if not pd.isnull(oDFrame_VALUE):
if oDFrame_VALUE < oElem_DATA_VALUES:
oDict_DATA_SEARCH[sElem_DATA_COLS] = None
if sDFrame_TYPE == 'observed':
oDict_DATA_VALUES = list(oDict_DATA_SEARCH.values())
for sElem_DATA_COLS, oElem_DATA_VALUES in zip(
oDFrame_COLS, oDict_DATA_VALUES):
if oElem_DATA_VALUES is not None:
if not pd.isnull(oDFrame_VALUE):
if oDFrame_VALUE > oElem_DATA_VALUES:
oDict_DATA_SEARCH[sElem_DATA_COLS] = None
if sDFrame_TYPE == 'forecast':
oDict_DATA_VALUES = list(oDict_DATA_SEARCH.values())
for sElem_DATA_COLS, oElem_DATA_VALUES in zip(
oDFrame_COLS, oDict_DATA_VALUES):
if oElem_DATA_VALUES is not None:
if not pd.isnull(oDFrame_VALUE):
if oDFrame_VALUE > oElem_DATA_VALUES:
oDict_DATA_SEARCH[sElem_DATA_COLS] = None
if None not in oDict_DATA_SEARCH.values():
oDFrame_DATA_SEARCH = oDFrame_DATA_SEARCH.append(
pd.DataFrame(oDict_DATA_SEARCH, index=oVarTIME_ROW))
oDict_DATA_SEARCH = None
# break
# Append value for last part to get also not completed dictionary
if oDict_DATA_SEARCH is not None:
if any(oElem is not None for oElem in oDict_DATA_SEARCH.values()):
oDFrame_DATA_SEARCH = oDFrame_DATA_SEARCH.append(
pd.DataFrame(oDict_DATA_SEARCH, index=oVarTIME_ROW))
# Delete rows with all nan(s)
oDFrame_DATA_ALL = oDFrame_DATA_SEARCH.dropna(axis=0, how='all')
oDFrame_DATA_FINITE = oDFrame_DATA_SEARCH.dropna(axis=0, how='any')
if not oDFrame_DATA_FINITE.empty:
oDFrame_DATA_SELECT = oDFrame_DATA_FINITE
else:
Exc.getExc(
' =====> WARNING: source and/or ancillary columns is/are null!', 2,
1)
oDFrame_DATA_SELECT = oDFrame_DATA_ALL
# Initialize time dictionary (if none is actual value)
if oVarTIME_MEMORY is None:
oVarTIME_MEMORY = {}
# Iterate over time select
for oDFrame_ROW_SELECT in oDFrame_DATA_SELECT.iterrows():
# Iterate over columns of SRC and ANC name(s)
for sDFrame_COLS, sDFrame_TYPE in zip(oDFrame_COLS, oDFrame_TYPE):
# Check data availability
if oDFrame_ROW_SELECT:
# Get selected time
oVarTIME_ROW_SELECT = oDFrame_ROW_SELECT[1][sDFrame_COLS]
if sDFrame_COLS in list(oVarTIME_MEMORY.keys()):
oVarFILE_TIMESAVED = oVarTIME_MEMORY[sDFrame_COLS]
# oVarFILE_TIMEACTUAL = pd.Timestamp(oDFrame_ROW_SELECT[sDFrame_COLS].values[0])
oVarFILE_TIMEACTUAL = oVarTIME_ROW_SELECT
if oVarFILE_TIMEACTUAL > oVarFILE_TIMESAVED:
oVarTIME_MEMORY[sDFrame_COLS] = None
oVarFILE_MEMORY[sDFrame_COLS] = False
# Check to update path(s) only one time
if not oVarFILE_MEMORY[sDFrame_COLS]:
sVarFILE_PATHSELECT = oVarFILE_PATHGENERIC[sDFrame_COLS]
# oVarFILE_TIMESELECT = pd.Timestamp(oDFrame_DATA_SELECT[sDFrame_COLS].values[0])
oVarFILE_TIMESELECT = oVarTIME_ROW_SELECT
oListFILE_TAGS = setDatasetTags_Finder(
oVarFILE_TIMESELECT)[0]
sVarFILE_PATHBASE, sVarFILE_FILEBASE = os.path.split(
sVarFILE_PATHSELECT)
if sDFrame_TYPE == 'observed':
sVarFILE_PATHREF = join(sVarFILE_PATHBASE,
sVarFILE_FILEBASE)
elif (sDFrame_TYPE == 'forecast') or (sDFrame_TYPE
== 'result'):
sVarFILE_PATHDEF = defineString(
sVarFILE_PATHBASE, oListFILE_TAGS)
sVarFILE_PATHREF = join(sVarFILE_PATHDEF,
sVarFILE_FILEBASE)
oVarTIME_MEMORY[sDFrame_COLS] = oVarFILE_TIMESELECT
oVarFILE_PATHREF[sDFrame_COLS] = sVarFILE_PATHREF
oVarFILE_MEMORY[sDFrame_COLS] = True
else:
Exc.getExc(
' =====> WARNING: dataframe for ' + sDFrame_COLS +
' variable is null!', 2, 1)
oVarTIME_MEMORY[sDFrame_COLS] = oVarRUN_TIMESTEP
oVarFILE_MEMORY[sDFrame_COLS] = True
return oVarTIME_MEMORY, oVarFILE_PATHREF, oVarFILE_MEMORY
def main():
# -------------------------------------------------------------------------------------
# Version and algorithm information
sProgramVersion = '1.2.0'
sProjectName = 'HAT'
sAlgType = 'Analyzer_Datasets'
sAlgName = 'HMC NearRealTime'
# Time algorithm information
dStartTime = time.time()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Get script argument(s)
[sScriptName, sFileSetting, sTimeArg] = GetArgs()
# Set algorithm configuration
oDrv_Data_Settings = DataAlgorithm(sFileSetting)
[oData_Settings, oData_Path, oData_Flags, oData_Mapping,
oData_ColorMap] = oDrv_Data_Settings.getDataSettings()
# Set logging file
oLogStream = setLoggingFile(
oData_Path['log'],
bLoggerHistory=oData_Settings['data']['log']['history'])
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Start Program
oLogStream.info('[' + sProjectName + ' ' + sAlgType + ' - ' + sAlgName +
' (Version ' + sProgramVersion + ')]')
oLogStream.info('[' + sProjectName + '] Start Program ... ')
# Clean static ancillary file(s) and folder(s)
DataAnalysisCleaner(
flag=[oData_Flags['cleaning_static_data']],
file=[oData_Path['section_ancillary'],
oData_Path['land_ancillary']]).cleanDataAnalysis()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Get algorithm time
oLogStream.info(' --> Set algorithm time ... ')
oDrv_Time = DataTime(
sTimeArg,
sTimeNow=oData_Settings['time']['time_now'],
iTimePeriodPast=oData_Settings['time']['time_period'],
sTimeFrequency=oData_Settings['time']['time_frequency'],
sTimeETA=oData_Settings['time']['time_eta'])
oData_TimeRun = oDrv_Time.getDataTime(bTimeReverse=True,
bTimeRestart=False)
oLogStream.info(' --> Set algorithm time ... DONE')
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Get geographical data
oLogStream.info(' --> Set geographical data ... ')
oDrv_Data_Geo = DataGeo(
file_terrain=oData_Path['land_terrain'],
file_channel_network=oData_Path['land_channel_network'],
file_ancillary=oData_Path['land_ancillary'],
vars=['Terrain', 'Channel_Network'])
oData_Geo = oDrv_Data_Geo.getDataGeo()
oLogStream.info(' --> Set geographical data ... DONE')
# Get sections data
oLogStream.info(' --> Set section data ... ')
oDrv_Data_Section = DataPoint(
file_shp=oData_Path['section_ref'],
file_mask=oData_Path['section_mask'],
file_terrain=oData_Path['section_dem'],
file_ctime=oData_Path['section_ctime'],
file_ancillary=oData_Path['section_ancillary'],
fields=None,
)
oData_Section = oDrv_Data_Section.getDataPoint(oData_Geo)
oLogStream.info(' --> Set section data ... DONE')
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Iterate over time steps
for oTimeRun_STEP in oData_TimeRun['oTimeSteps']:
# -------------------------------------------------------------------------------------
# Get data time
oLogStream.info(' --> Get dynamic time ... ')
oDrv_DataTime_Dynamic = DataAnalysisTime(time=oTimeRun_STEP,
settings=oData_Settings)
oData_TimePeriod = oDrv_DataTime_Dynamic.computeDataTime(oData_Geo)
oLogStream.info(' --> Get dynamic time ... DONE')
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Set data tags /$yyyy/$mm/$dd/
oDrv_Data_Tags = DataTags({
'$yyyy':
str(oTimeRun_STEP.year).zfill(4),
'$mm':
str(oTimeRun_STEP.month).zfill(2),
'$dd':
str(oTimeRun_STEP.day).zfill(2),
'$HH':
str(oTimeRun_STEP.hour).zfill(2),
'$MM':
str(oTimeRun_STEP.minute).zfill(2),
'$RUNTIME':
str(oTimeRun_STEP.year).zfill(4) + '/' +
str(oTimeRun_STEP.month).zfill(2) + '/' +
str(oTimeRun_STEP.day).zfill(2) + '/' +
str(oTimeRun_STEP.hour).zfill(2),
'$RUNDOMAIN':
oData_Settings['algorithm']['ancillary']['run_domain'],
'$RUNNAME':
oData_Settings['algorithm']['ancillary']['run_name']
})
oData_Tags = oDrv_Data_Tags.setDataTags()
# Info time
oLogStream.info(' --> Set time step: ' + str(oTimeRun_STEP))
# Clean buffer and product file(s) and folder(s)
DataAnalysisCleaner(
flag=[
oData_Flags['cleaning_dynamic_data_source'],
oData_Flags['cleaning_dynamic_data_outcome'],
],
file=[[
defineString(oData_Path['file_buffer_ts'], oData_Tags),
defineString(oData_Path['file_buffer_gridded'], oData_Tags),
defineString(oData_Path['file_buffer_info'], oData_Tags)
],
[
defineString(oData_Path['file_buffer_registry'],
oData_Tags),
[
defineString(oData_Path[sListItem], oData_Tags)
for sListItem in list(
oData_Mapping['file_dewetra_data_ts'])
],
[
defineString(oData_Path[sListItem], oData_Tags)
for sListItem in list(
oData_Mapping['file_dewetra_data_gridded'])
],
[
defineString(oData_Path[sListItem], oData_Tags)
for sListItem in list(
oData_Mapping['file_hydrapp_graph_ts'])
],
[
defineString(oData_Path[sListItem], oData_Tags)
for sListItem in list(
oData_Mapping['file_hydrapp_graph_gridded'])
],
]]).cleanDataAnalysis()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Get data analysis
oLogStream.info(' --> Seek analysis data ... ')
oDrv_DataSeeker_Dynamic = DataAnalysisSeeker(
oData_Settings['algorithm']['ancillary']['run_application'],
time_seek=oTimeRun_STEP,
time_period=oData_TimePeriod,
settings=oData_Settings,
file=oData_Path,
mapping=oData_Mapping,
tags=oData_Tags)
if not exists(defineString(oData_Path['file_buffer_info'],
oData_Tags)):
oData_Dynamic_Source_Info = oDrv_DataSeeker_Dynamic.seekDataAnalysis(
oData_Section)
savePickle(
defineString(oData_Path['file_buffer_info'], oData_Tags),
oData_Dynamic_Source_Info)
oLogStream.info(' --> Seek dynamic data ... DONE')
else:
oData_Dynamic_Source_Info = restorePickle(
defineString(oData_Path['file_buffer_info'], oData_Tags))
oLogStream.info(
' --> Seek dynamic data ... LOADED from workspace saved file.')
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Make data analysis
oLogStream.info(' --> Make analysis data... ')
oDrv_DataMaker_Dynamic = DataAnalysisMaker(
oData_Settings['algorithm']['ancillary']['run_application'],
time_make=oTimeRun_STEP,
time_period=oData_TimePeriod,
settings=oData_Settings,
file=oData_Path,
mapping=oData_Mapping,
cmap=oData_ColorMap,
tags=oData_Tags)
if not exists(
defineString(oData_Path['file_buffer_registry'], oData_Tags)):
oData_Dynamic_Outcome_Info = oDrv_DataMaker_Dynamic.selectDataAnalysisMaker(
oData_Dynamic_Source_Info, oData_Geo, oData_Section)
savePickle(
defineString(oData_Path['file_buffer_registry'], oData_Tags),
oData_Dynamic_Outcome_Info)
oLogStream.info(' --> Make analysis data ... DONE')
else:
oLogStream.info(
' --> Make analysis data ... SKIPPED! Data previously published.'
)
oData_Dynamic_Outcome_Info = restorePickle(
defineString(oData_Path['file_buffer_registry'], oData_Tags))
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Clean buffer file(s) and folder(s)
DataAnalysisCleaner(
flag=[
oData_Flags['cleaning_dynamic_data_source'],
oData_Flags['cleaning_dynamic_data_outcome']
],
file=[[
defineString(oData_Path['file_buffer_ts'], oData_Tags),
defineString(oData_Path['file_buffer_gridded'], oData_Tags),
defineString(oData_Path['file_buffer_info'], oData_Tags)
], [defineString(oData_Path['file_buffer_registry'],
oData_Tags)]]).cleanDataAnalysis()
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Note about script parameter(s)
oLogStream.info('NOTE - Algorithm parameter(s)')
oLogStream.info('Script: ' + str(sScriptName))
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# End Program
dTimeElapsed = round(time.time() - dStartTime, 1)
oLogStream.info('[' + sProjectName + ' ' + sAlgType + ' - ' + sAlgName +
' (Version ' + sProgramVersion + ')]')
oLogStream.info('End Program - Time elapsed: ' + str(dTimeElapsed) +
' seconds')
Exc.getExc('', 0, 0)
def defineDataTags(sString, oTags):
return defineString(sString, oTags)
def __savePointInfo(self, oDataPoint, oDataGeo):
# -------------------------------------------------------------------------------------
# Initialise section workspace
oDataPoint['SEC_FILE_ANCILLARY'] = None
oDataPoint['SEC_FILE_TERRAIN'] = None
oDataPoint['SEC_FILE_CTIME'] = None
oDataPoint['SEC_FILE_MASK'] = None
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Iterate over section(s)
for iSectionID, oSectionPoint in enumerate(oDataPoint.iterrows()):
# -------------------------------------------------------------------------------------
# Define section info
oSectionData = oSectionPoint[1]
sSectionName = oSectionData['SEC_TAG']
sSectionName = sSectionName.lower().replace(':', '')
# Info start
oLogStream.info(' ----> Save point info for section ' +
sSectionName + ' ... ')
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Define filename(s)
sFileName_MASK = defineString(self.__sFileName_MASK,
{'$section': sSectionName})
sFileName_TERRAIN = defineString(self.__sFileName_TERRAIN,
{'$section': sSectionName})
sFileName_CTIME = defineString(self.__sFileName_CTIME,
{'$section': sSectionName})
sFileName_ANC = defineString(self.__sFileName_ANC,
{'$section': sSectionName})
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Check ancillary section file availability
if not isfile(sFileName_ANC):
# -------------------------------------------------------------------------------------
# Get mask information
if isfile(sFileName_MASK):
# Open ascii grid data
oFileDriver = Drv_Data_IO(sFileName_MASK).oFileWorkspace
oFileData = oFileDriver.oFileLibrary.openFile(
join(oFileDriver.sFilePath, oFileDriver.sFileName),
'r')
# Read ascii grid data
[a2iVarMask, a1oHeaderMask
] = oFileDriver.oFileLibrary.readArcGrid(oFileData)
a2iVarMask = np.flipud(a2iVarMask)
else:
Exc.getExc(
' =====> WARNING: file mask undefined (' +
sFileName_MASK + ')', 2, 1)
a2iVarMask = None
a1oHeaderMask = None
# Get terrain information
if isfile(sFileName_TERRAIN):
# Open ascii grid data
oFileDriver = Drv_Data_IO(sFileName_TERRAIN).oFileWorkspace
oFileData = oFileDriver.oFileLibrary.openFile(
join(oFileDriver.sFilePath, oFileDriver.sFileName),
'r')
# Read ascii grid data
[a2dVarTerrain, a1oHeaderTerrain
] = oFileDriver.oFileLibrary.readArcGrid(oFileData)
a2dVarTerrain = np.flipud(a2dVarTerrain)
else:
Exc.getExc(
' =====> WARNING: file terrain undefined (' +
sFileName_TERRAIN + ')', 2, 1)
a2dVarTerrain = None
a1oHeaderTerrain = None
# Get corrivation time information
if isfile(sFileName_CTIME):
# Open ascii grid data
oFileDriver = Drv_Data_IO(sFileName_CTIME).oFileWorkspace
oFileData = oFileDriver.oFileLibrary.openFile(
join(oFileDriver.sFilePath, oFileDriver.sFileName),
'r')
# Read ascii grid data
[a2dVarCorrTime, a1oHeaderCorrTime
] = oFileDriver.oFileLibrary.readArcGrid(oFileData)
a2dVarCorrTime = np.flipud(a2dVarCorrTime)
else:
Exc.getExc(
' =====> WARNING: file corrivation_time undefined (' +
sFileName_CTIME + ')', 2, 1)
a2dVarCorrTime = None
a1oHeaderCorrTime = None
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Check mask, terrain and corrivation time information
if (a2iVarMask
is not None) and (a2dVarCorrTime
is not None) and (a2dVarTerrain
is not None):
# -------------------------------------------------------------------------------------
# Read geographical header
[
iRows, iCols, dGeoXMin, dGeoYMin, dGeoXStep, dGeoYStep,
dNoData
] = readGeoHeader(a1oHeaderTerrain)
# Define geographical corners
[dGeoXMin, dGeoXMax, dGeoYMin,
dGeoYMax] = defineGeoCorner(dGeoXMin, dGeoYMin, dGeoXStep,
dGeoYStep, iCols, iRows)
# Define geographical grid
[a2dGeoX, a2dGeoY,
a1dGeoBox] = defineGeoGrid(dGeoYMin, dGeoXMin, dGeoYMax,
dGeoXMax, dGeoYStep, dGeoXStep)
# Compute section drainage area
a2dCellArea = computeCellArea(a2dGeoY, dGeoXStep,
dGeoYStep)
dCellArea = np.nanmean(a2dCellArea)
dDrainageArea = computeDrainageArea(
a2dVarTerrain, dCellArea)
# Add drainage area to attribute(s)
oSectionData['DrainageArea'] = dDrainageArea
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Define encoding
oGeoEncoding = {
'Mask': {
'_FillValue': a1oHeaderMask['NODATA_value']
},
'Terrain': {
'_FillValue': a1oHeaderTerrain['NODATA_value']
},
'CorrTime': {
'_FillValue': a1oHeaderCorrTime['NODATA_value']
}
}
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Create data array for mask
oVarMask = createVar2D(a2iVarMask,
oDataGeo.a2dGeoX,
oDataGeo.a2dGeoY,
sVarName='Mask')
# Create data array for terrain
oVarTerrain = createVar2D(a2dVarTerrain,
oDataGeo.a2dGeoX,
oDataGeo.a2dGeoY,
sVarName='Terrain')
# Create data array for channel network
oVarCorrTime = createVar2D(a2dVarCorrTime,
oDataGeo.a2dGeoX,
oDataGeo.a2dGeoY,
sVarName='CorrTime')
# Merge data arrays in a common dataset
oVarDSet = mergeVar2D(
[oVarMask, oVarTerrain, oVarCorrTime])
# Add attribute to section dataset
for oSectionKey, oSectionValue in oSectionData.items():
if oSectionKey != 'geometry':
if oSectionValue is not None:
oVarDSet.attrs[oSectionKey] = oSectionValue
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Save data in a netcdf file
if not isfile(sFileName_ANC):
createFolderByFile(sFileName_ANC)
sVarMode = 'w'
else:
sVarMode = 'a'
# Dump data to netcdf file
writeFileNC4(sFileName_ANC,
oVarDSet,
oVarAttr=oGeoEncoding,
sVarGroup=None,
sVarMode=sVarMode,
oVarEngine='h5netcdf',
iVarCompression=9)
# Info end
oLogStream.info(' ----> Save point info for section ' +
sSectionName + ' ... OK')
# -------------------------------------------------------------------------------------
else:
# -------------------------------------------------------------------------------------
# Info exit
Exc.getExc(
' =====> WARNING: not enough information to save info section file',
2, 1)
oLogStream.info(' ----> Save point info for section ' +
sSectionName + ' ... FAILED')
# -------------------------------------------------------------------------------------
else:
# -------------------------------------------------------------------------------------
# Info skip
oLogStream.info(
' ----> Save point info ... previously done. SKIPPED')
# Get data processed previously to extract some information
oFileHandle = readFileNC4(sFileName_ANC)
if hasattr(oFileHandle, 'DrainageArea'):
dDrainageArea = getattr(oFileHandle, 'DrainageArea')
else:
dDrainageArea = -9999.0
Exc.getExc(
' =====> WARNING: drainage area value not available! Set to -9999.0',
2, 1)
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Update data point with defined filename
oDataPoint.at[iSectionID, 'OUTLET_FILE_ANCILLARY'] = sFileName_ANC
oDataPoint.at[iSectionID,
'OUTLET_FILE_TERRAIN'] = sFileName_TERRAIN
oDataPoint.at[iSectionID, 'OUTLET_FILE_CTIME'] = sFileName_CTIME
oDataPoint.at[iSectionID, 'OUTLET_FILE_MASK'] = sFileName_MASK
oDataPoint.at[iSectionID, 'OUTLET_NAME'] = sSectionName
oDataPoint.at[iSectionID, 'DrainageArea'] = dDrainageArea
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Sort data point using BASIN and DrainageArea as pivot(s)
oDataPoint_SORT = oDataPoint.sort_values(['BASIN', 'DrainageArea'])
oDataPoint_SORT['HAT_ELEMENT'] = oDataPoint_SORT.groupby(
'BASIN')['BASIN'].transform('count')
oDataPoint_SORT["HAT_ID_ROWS"] = oDataPoint_SORT['BASIN'].map({
iX: iI
for iI, iX in enumerate(oDataPoint_SORT['BASIN'].unique())
}) + 1
oDataPoint_SORT["HAT_ID_COLS"] = oDataPoint_SORT.groupby(
"BASIN").cumcount() + 1
# -------------------------------------------------------------------------------------
# -------------------------------------------------------------------------------------
# Return update points workspace
return oDataPoint_SORT