def test_extract(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=5,
base_year=1870)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/sunspot_area.csv')
ret1 = wdmtoolbox.extract(self.wdmname, 101).astype('f')
ret2 = wdmtoolbox.extract('{0},101'.format(self.wdmname)).astype('f')
assert_frame_equal(ret1, ret2)
ret3 = tstoolbox.read('tests/sunspot_area.csv').astype('f')
ret1.columns = ['Area']
assert_frame_equal(ret1, ret3)
ret4 = tstoolbox.read('tests/sunspot_area_with_missing.csv',
dropna='no').astype('f')
wdmtoolbox.createnewdsn(self.wdmname,
500,
tcode=5,
base_year=1870)
wdmtoolbox.csvtowdm(self.wdmname,
500,
input_ts='tests/sunspot_area_with_missing.csv')
ret5 = wdmtoolbox.extract(self.wdmname, 500).astype('f')
ret5.columns = ['Area']
assert_frame_equal(ret5, ret4)
def test_tstep(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2, base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101, input_ts="tests/nwisiv_02246000.csv")
ret1 = wdmtoolbox.extract(self.wdmname, 101)
ret2 = wdmtoolbox.extract("{0},101".format(self.wdmname))
assert_frame_equal(ret1, ret2, check_index_type=False)
ret3 = tstoolbox.read("tests/nwisiv_02246000.csv").astype("float64")
ret3.index = ret3.index.tz_localize(None)
ret1.columns = ["02246000_iv_00060"]
assert_frame_equal(ret1, ret3, check_index_type=False)
def test_tstep(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
ret1 = wdmtoolbox.extract(self.wdmname, 101)
ret2 = wdmtoolbox.extract('{0},101'.format(self.wdmname))
assert_frame_equal(ret1, ret2)
ret3 = tstoolbox.read('tests/nwisiv_02246000.csv').astype('float64')
ret1.columns = ['02246000_iv_00060']
assert_frame_equal(ret1, ret3)
def test_copy_to_self(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
ret1 = wdmtoolbox.extract(self.wdmname, 101)
ret2 = wdmtoolbox.extract('{0},101'.format(self.wdmname))
assert_frame_equal(ret1, ret2)
ret3 = tstoolbox.read('tests/nwisiv_02246000.csv').astype('float64')
ret1.columns = ['02246000_iv_00060']
assert_frame_equal(ret1, ret3)
wdmtoolbox.copydsn(self.wdmname, 101, self.wdmname, 1101)
def test_out_of_bounds_dsn(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2, base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101, input_ts="tests/nwisiv_02246000.csv")
with self.assertRaisesRegex(
WDMError, "WDM error: data set number out of valid range"
):
ret1 = wdmtoolbox.extract(self.wdmname, 32001)
def test_dsn_not_in_wdm(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
with assertRaisesRegexp(WDMError, 'error code -81'):
ret1 = wdmtoolbox.extract(self.wdmname, 32000)
def test_extract_args(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
with assertRaisesRegexp(ValueError,
'The only allowed keywords are'):
ret1 = wdmtoolbox.extract(self.wdmname, 101, ph=True)
def test_out_of_bounds_dsn(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
with assertRaisesRegexp(WDMError,
'WDM error: data set number out of valid range'):
ret1 = wdmtoolbox.extract(self.wdmname, 32001)
def test_start_date(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
ret1 = wdmtoolbox.extract(self.wdmname, 101, start_date='2014-02-21 16:00:00')
ret3 = tstoolbox.read('tests/nwisiv_02246000.csv', start_date='2014-02-21 16:00:00').astype('float64')
ret1.columns = ['02246000_iv_00060']
assert_frame_equal(ret1, ret3)
def test_dsn_not_in_wdm(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname,
101,
tcode=2,
base_year=1970,
tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname,
101,
input_ts='tests/nwisiv_02246000.csv')
with self.assertRaisesRegexp(WDMError, 'error code -81'):
ret1 = wdmtoolbox.extract(self.wdmname, 32000)
def test_negative_dsn(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname,
101,
tcode=2,
base_year=1970,
tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname,
101,
input_ts='tests/nwisiv_02246000.csv')
with self.assertRaisesRegexp(
WDMError, 'WDM error: data set number out of valid range'):
ret1 = wdmtoolbox.extract(self.wdmname, 0)
def test_extract_args(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname,
101,
tcode=2,
base_year=1970,
tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname,
101,
input_ts='tests/nwisiv_02246000.csv')
with self.assertRaisesRegexp(ValueError,
'The only allowed keywords are'):
ret1 = wdmtoolbox.extract(self.wdmname, 101, ph=True)
def test_extract(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=5, base_year=1870)
wdmtoolbox.csvtowdm(self.wdmname, 101, input_ts="tests/sunspot_area.csv")
ret1 = wdmtoolbox.extract(self.wdmname, 101).astype("f")
ret2 = wdmtoolbox.extract("{0},101".format(self.wdmname)).astype("f")
assert_frame_equal(ret1, ret2)
ret3 = tstoolbox.read("tests/sunspot_area.csv").astype("f")
ret1.columns = ["Area"]
assert_frame_equal(ret1, ret3)
ret4 = tstoolbox.read(
"tests/sunspot_area_with_missing.csv", dropna="no"
).astype("f")
wdmtoolbox.createnewdsn(self.wdmname, 500, tcode=5, base_year=1870)
wdmtoolbox.csvtowdm(
self.wdmname, 500, input_ts="tests/sunspot_area_with_missing.csv"
)
ret5 = wdmtoolbox.extract(self.wdmname, 500).astype("f")
ret5.columns = ["Area"]
assert_frame_equal(ret5, ret4)
def test_extract(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=5, base_year=1870)
wdmtoolbox.csvtowdm(self.wdmname,
101,
input_ts='tests/sunspot_area.csv')
ret1 = wdmtoolbox.extract(self.wdmname, 101)
ret2 = wdmtoolbox.extract('{0},101'.format(self.wdmname))
assert_frame_equal(ret1, ret2)
ret3 = tstoolbox.read('tests/sunspot_area.csv')
ret1.columns = ['Area']
assert_frame_equal(ret1, ret3)
ret4 = tstoolbox.read('tests/sunspot_area_with_missing.csv',
dropna='no')
wdmtoolbox.createnewdsn(self.wdmname, 500, tcode=5, base_year=1870)
wdmtoolbox.csvtowdm(self.wdmname,
500,
input_ts='tests/sunspot_area_with_missing.csv')
ret5 = wdmtoolbox.extract(self.wdmname, 500)
ret5.columns = ['Area']
assert_frame_equal(ret5, ret4)
def test_end_date(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2,
base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101,
input_ts='tests/nwisiv_02246000.csv')
ret1 = wdmtoolbox.extract(self.wdmname,
101,
end_date='2014-02-22 11:00:00')
ret3 = tstoolbox.read('tests/nwisiv_02246000.csv',
end_date='2014-02-22 11:00:00').astype('float64')
ret3.index = ret3.index.tz_localize(None)
ret1.columns = ['02246000_iv_00060']
assert_frame_equal(ret1, ret3, check_index_type=False)
def test_end_date(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname,
101,
tcode=2,
base_year=1970,
tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname,
101,
input_ts='tests/nwisiv_02246000.csv')
ret1 = wdmtoolbox.extract(self.wdmname,
101,
end_date='2014-02-22 11:00:00')
ret3 = tstoolbox.read('tests/nwisiv_02246000.csv',
end_date='2014-02-22 11:00:00').astype('float64')
ret1.columns = ['02246000_iv_00060']
assert_frame_equal(ret1, ret3)
def test_dates(self):
wdmtoolbox.createnewwdm(self.wdmname, overwrite=True)
wdmtoolbox.createnewdsn(self.wdmname, 101, tcode=2, base_year=1970, tsstep=15)
wdmtoolbox.csvtowdm(self.wdmname, 101, input_ts="tests/nwisiv_02246000.csv")
ret1 = wdmtoolbox.extract(
self.wdmname,
101,
start_date="2014-02-21 16:00:00",
end_date="2014-02-22 11:00:00",
)
ret3 = tstoolbox.read(
"tests/nwisiv_02246000.csv",
start_date="2014-02-21 16:00:00",
end_date="2014-02-22 11:00:00",
).astype("float64")
ret3.index = ret3.index.tz_localize(None)
ret1.columns = ["02246000_iv_00060"]
assert_frame_equal(ret1, ret3, check_index_type=False)
'''This is an example script that finds all the DSN from a WDM file
and uses the first DSN to extract the data and plot it.
It can easily be modified to loop through all the datasets.'''
from wdmtoolbox import wdmtoolbox as wdm
import pandas as pd
files_path = 'C:\BASINS45\modelout\Test'
wdmfile = os.path.join(files_path, 'Test.wdm')
DataList = wdm.listdsns(
wdmfile) #Getting the list of all datasets from the WDMFile as OrderedDict
DSNList = list(DataList.keys()) #Extracting the list of DSN from DataList
dataSeries = wdm.extract(
wdmfile,
DSNList[0]) #extracting the dataset using the first DSN in DSNList
dataSeries = dataSeries['2002-05-31':'2002-07-31'] #subsetting the data
dataSeries.plot() #plotting the dataframe
#dataSeries is a dataframe
allFiles1 = glob.glob(source_dir_met + "/*met*.wdm*")
allFiles2 = glob.glob(source_dir_prad + "/*prad*.wdm*")
#our file destination
output_folder = r"C:\Users\icprbadmin\Documents\Python_Scripts\FEWS_WDM\output"
#print("win32 os")
#---------------------------------------------------#
#------------iterate through the met files---#
for file in allFiles1:
#------------1000.PET-------------#
# takes specific indexes from the right to include only the unique identification code
title = file[-10:-4]
# use wdmtoolbox function to assign data to a pandas dataframe
data_df = wdmtoolbox.extract(file, 1000)
#add rows to dataframe to accommodate year, month, day and hour we will pull from our datetimeindex object
data_df = pd.concat(
[data_df,
pd.DataFrame(columns=("year", "month", "day", "hour"))],
sort=False)
#then we add the values from the DatetimeIndex to the appropriate column
data_df['year'] = data_df.index.year
data_df['month'] = data_df.index.month
data_df['day'] = data_df.index.day
data_df['hour'] = data_df.index.hour
#output the data to the output directory with the required file extension
data_df.to_csv(output_folder + "/" + title + ".PET")
#-----------------------------------#
#------------1001.DPT-----------------#
if 'End of Job' in echoFileLines[len(echoFileLines) - 1]:
print('Real Time HSPF model ran successfully!')
else:
print('Real Time HSPF model did not run successfully')
sys.exit()
except:
print('Could not open echo file. Check if any of the files are locked.')
sys.exit()
DSNList = [100]
#A list of DSN that are in the HSPF Model Output WDM file with the data that is needed for the EFDC+ model
OverAllDF = []
for DSN in DSNList:
#Read the WDM data and save in a dataframe
df = wdm.extract(os.path.join(pHSPFModelPath, pWDMFile), DSN)
df = df['2012-1-1':]
#Clip the data
df = df.resample('D').mean()
#resample the data to daily
df.columns = ['HSPF_' + str(DSN)]
#Rename the column name
diff = df.index - zeroDay
df.index = diff / pd.to_timedelta(1, unit='D')
#Change the index so that it matches the numerical indexing for EFDC+
OverAllDF.append(df)
OverAllDF = pd.concat(OverAllDF, axis=1)
#The OverAllDF has all the HSPF model outputs and it can be processed to
#add in the EFDC+ model.
'''Now, Read the existing qser.inp file and append the HSPF data to it.
# print todaysdate
threemonthago = datetime.datetime(correctyear, correctmonth, i.day)
# print threemonthago
# ---------------------------------------------------------------------------#
# ---going through the met files and prnt files with issues------------------#
# print 'Check the following files for negative values in wdms'
for file_ in allFiles1:
title = file_[-10:-4]
oldfile_ = glob.glob(metdirectory2 + "/met_" + title + ".wdm")
figurename1 = path1 + "/atmp/atmp_" + title + ".png"
# print oldfile_[0]
figurename2 = path1 + "/atmp/old_atmp_" + title + ".png"
dataatmp1 = wdmtoolbox.extract(file_, 1004)
dataatmp2 = wdmtoolbox.extract(oldfile_[0], 1004)
df1 = dataatmp1
df2 = dataatmp2
df3 = df2.subtract(df1, fill_value=0)
a = 1
# textfile = path1+ "/atmp/atmp_" + title + ".txt"
if a == 1:
# print figurename1
# f1=open(textfile)
# f1=textfile
# nv.to_csv(f1)
# f1.close()
fig, [ax1, ax2, ax3] = plt.subplots(nrows=3, ncols=1)
fig.set_size_inches(10.5, 10.5)
from wdmtoolbox import wdmtoolbox as wdm
import pandas as pd
"""Locate the wdm file"""
wdmfile = 'C:\BASINS45\modelout\LowWill_APGIS\LowerWil.wdm'
DataList = wdm.listdsns(wdmfile)
RCHRES_List = [
'R:101', 'R:103', 'R:105', 'R:107', 'R:109', 'R:111', 'R:113', 'R:115',
'R:117', 'R:119', 'R:121', 'R:123', 'R:125', 'R:127', 'R:129', 'R:131',
'R:133', 'R:135', 'R:137', 'R:139', 'R:141', 'R:143', 'R:201', 'R:203',
'R:205', 'R:207', 'R:209', 'R:211', 'R:213', 'R:215', 'R:217', 'R:219',
'R:221', 'R:223', 'R:225', 'R:227', 'R:229', 'R:231', 'R:233', 'R:235',
'R:237', 'R:301', 'R:303', 'R:305', 'R:307', 'R:309', 'R:401', 'R:403',
'R:405', 'R:407', 'R:409', 'R:411', 'R:413', 'R:415', 'R:417'
]
TauAnalysisText = ''
for key in DataList.keys():
data = DataList[key]
Location = data['location'].decode('utf-8')
Constituent = data['constituent'].decode('utf-8')
if Constituent == 'TAU':
dataSeries = wdm.extract(wdmfile, key)
TauAnalysisText+= Location +','+ '{:.3E}'.format(dataSeries.quantile(0.01)[0]) + \
','+ '{:.3E}'.format(dataSeries.quantile(0.02)[0]) + \
','+ '{:.3E}'.format(dataSeries.quantile(0.98)[0]) + \
','+ '{:.3E}'.format(dataSeries.quantile(0.99)[0]) + '\n'
f = open('C:\BASINS45\modelout\LowWill_APGIS\LowWillTau.txt', 'w+')
f.write(TauAnalysisText)
f.close()
