def _read_dss_values(alternative_basename: str,
reservoir_id: str,
base_dir: str,
start_date: str = "01JAN2001 00:00:00",
end_date: str = "30JUL2001 00:00:00"):
"""
Parameters
----------
alternative_name
reservoir_id
variable_type
base_dir
start_date
end_date
Returns
-------
"""
base_name = os.path.basename(base_dir)
# TODO : don't hardcode
dss_file = os.path.join(
base_dir, 'base/Outaouais_long/shared/{}'.format(alternative_basename))
dss_filename_output = os.path.join(
base_dir, 'base/Outaouais_long/rss/simulation/simulation.dss')
alternative_name = "{:07d}".format(
int(base_name[1:]) * 100 - 100 +
int(alternative_basename.split('.')[0]))
pathname = "/{}/{}///1DAY/{}/".format(alternative_name, reservoir_id,
alternative_name)
fid = HecDss.Open(dss_file)
ts = fid.read_ts(pathname,
window=(start_date, end_date),
trim_missing=True)
values = ts.values
fid.close()
# Prepare time-series data
tsc = TimeSeriesContainer()
tsc.startDateTime = start_date
tsc.numberValues = len(values)
tsc.units = "cms"
tsc.type = "INST-VAL"
tsc.interval = 24 * 60
with HecDss.Open(dss_filename_output) as fid:
# add each column time-series from dataframe to hec
pathname = "/{}/{}///1DAY/{}/".format(
alternative_basename.split('.')[0], reservoir_id,
alternative_basename.split('.')[0])
tsc.pathname = pathname
tsc.values = values
# fid.deletePathname(tsc.pathname)
fid.put_ts(tsc)
def hmsrain(self, units='MM', type='PER-CUM'):
# input rainfall data into dss file
for stcode in self.stationNameList:
rainPathname = self.pathname(location=stcode, dss=True, flow=False)
shape = len(self.rainDict[stcode])
values = self.rainDict[stcode]
# write data into dss file
tsc = TimeSeriesContainer()
tsc.pathname = rainPathname
tsc.startDateTime = '{}:00:00'.format(
self.startTime.strftime('%d%b%Y %H'))
tsc.units = units
tsc.type = type
tsc.numberValues = shape
tsc.values = values
fid = HecDss.Open(self.rainfallPath)
fid.deletePathname(tsc.pathname)
fid.put_ts(tsc)
ts = fid.read_ts(rainPathname)
fid.close()
return
def test6():
# Fail
# Large date variation and seconds granularity although
# prevent_overflow = True tries to prevent overflow by setting smallest date (01JAN2019) as julianBaseDate
pathname = "/IRREGULAR/TIMESERIES/PARAM//IR-DAY/Ex14_Test6/"
T = ['01JAN2019 01:00', '01JAN2455 00:00']
tsc = TimeSeriesContainer()
tsc.pathname = pathname
tsc.interval = -1
tsc.granularity = 1
tsc.times = T
tsc.values = [2019, 5000]
tsc.numberValues = 2
fid.put_ts(tsc, prevent_overflow=True)
#
ts = fid.read_ts(pathname, regular=False)
print(ts.pytimes)
print(ts.values)
def test7():
# Pass
# Writing one time,value pair with prevent_overflow = True is safest
pathname = "/IRREGULAR/TIMESERIES/PARAM//IR-DECADE/Ex14_Test7/"
T = ['01JAN2019 02:01:05', '01JAN5000 01:02:06']
V = [2019, 5000]
for t, v in zip(T, V):
tsc = TimeSeriesContainer()
tsc.pathname = pathname
tsc.interval = -1
tsc.granularity = 60
tsc.times = [t]
tsc.values = [v]
tsc.numberValues = 1
fid.put_ts(tsc, prevent_overflow=True)
ts = fid.read_ts(pathname, regular=False)
print(ts.times)
print(ts.pytimes)
print(ts.values)
return tsc, ts
def gage2dss(output, site, param='flow', sdate='1700-01-01', edate=''):
"""Copy usgs flow or stage regular time-series data to dss file
Parameter
---------
output: string or HecDss.Open object
destination dss file path or object
site: string, list or dict
usgs gage id or list of ids or dictionary with key as description and value as gage id
param: string
flow or stage
sdate: string or datetime object
start date of the data
edate: string or datetime object
end date of the data
Returns
-------
None
Usage
-----
>>> gage2dss('dss_out.dss','08354900','flow','2000-01-01')
"""
if isinstance(site, (int, str)):
site_alias = [str(site)]
site_code = [str(site)]
elif isinstance(site, dict):
site_alias = list(site.keys())
site_code = list(site.values())
elif isinstance(site, (list, tuple)):
site_alias = [str(x) for x in site]
site_code = [str(x) for x in site]
else:
logging.error('Incorrect site argument value', exc_info=True)
_param = PARAM_ALLOWED.get(param.lower(), None)
if _param is None:
logging.error(
'Incorrect param argument value (%r). It mus be one of %r',
param,
PARAM_ALLOWED.keys(),
exc_info=True)
close_dss = False
if isinstance(output, HecDss.Open):
fidout = output
else:
fidout = HecDss.Open(output)
close_dss = True
if not edate:
edate = datetime.now().date().strftime('%Y-%m-%d')
try:
for alias, siteid in zip(site_alias, site_code):
url = NWIS_URL % (siteid, sdate, edate, _param)
logging.debug('url=%s', url)
req = urllib2.Request(url, None, headers=headers)
res = urllib2.urlopen(req)
data = res.read()
jdata = json.loads(data)
if not jdata['value']['timeSeries']:
logging.warning('No data for gage %s', siteid)
else:
ts = jdata['value']['timeSeries'][0]['values'][0]['value']
if not ts:
logging.warning('No data for gage %s', siteid)
else:
site = jdata['value']['timeSeries'][0]['sourceInfo'][
'siteName']
param_desc = (jdata['value']['timeSeries'][0]['variable']
['variableDescription']).split(',')[0]
unit = jdata['value']['timeSeries'][0]['variable']['unit'][
'unitCode']
logging.debug('Data param description is %s', param_desc)
logging.debug('Data unit is %s', unit)
nodata = ''
try:
nodata = jdata['value']['timeSeries'][0]['variable'][
'noDataValue']
except:
pass
# Determine time-series interval from first 10 data
dates = []
for row in ts[0:10]:
datestr = row['dateTime'].replace('T', ' ')
ptime = HecTime(datestr).python_datetime
dates.append(ptime)
date_diff = [y - x for y, x in zip(dates[1:], dates[0:-1])]
logging.debug('date_diff = %r', date_diff)
date_diff = [
x.total_seconds() / (60.0 * 60.0) for x in date_diff
]
period = min(date_diff)
logging.debug('Data interval = %r hour', period)
interval = '1DAY'
if period < 24:
interval = '1HOUR'
# Write to dss file
apart = ''
if alias != siteid:
apart = alias
bpart = site
cpart = param.upper()
dpart = ''
epart = interval
fpart = 'USGS ' + str(siteid)
pathname = '/%s/%s/%s/%s/%s/%s/' % (apart, bpart, cpart,
dpart, epart, fpart)
fidout.deletePathname(pathname)
tsc = TimeSeriesContainer()
tsc.pathname = pathname
tsc.type = 'INST'
tsc.units = unit
tsc.interval = 1
tsc.numberValues = 1
for row in ts:
date = row['dateTime'].replace('T', ' ')
# Filtering the No data values
if float(row['value']) == nodata:
flow = UNDEFINED
else:
flow = row['value']
tsc.startDateTime = date
tsc.values = [float(flow)]
fidout.put_ts(tsc)
except Exception as e:
logging.warning('Error occured for gage = %s', siteid)
raise e
finally:
if close_dss:
fidout.close()
'''
Write regular time-series data to example.dss
Notes:
The interval must be [any] integer greater than 0 for regular time-series.
Actual time-series interval implied from E-Part of pathname
The values attribute can be list, array or numpy array
'''
from datetime import datetime
from pydsstools.heclib.dss import HecDss
from pydsstools.core import TimeSeriesContainer, UNDEFINED
dss_file = "example.dss"
pathname = "/REGULAR/TIMESERIES/FLOW//1HOUR/Ex1/"
tsc = TimeSeriesContainer()
tsc.pathname = pathname
tsc.startDateTime = "15JUL2019 19:00:00"
tsc.numberValues = 5
tsc.units = "cfs"
tsc.type = "INST"
tsc.interval = 1
tsc.values = [100, UNDEFINED, 500, 5000, 10000]
fid = HecDss.Open(dss_file)
fid.deletePathname(tsc.pathname)
fid.put_ts(tsc)
ts = fid.read_ts(pathname)
fid.close()
def contrail2dss(**kwargs):
"""Copy irregular time-series data from Contrail Data Exchange to dss file
Parameter
---------
output: string or HecDss.Open object
destination dss file path or object
site_id: string
OneRain site id of gage
sensor_class: integer
integer value representing type of sensor such as rainfall, stage, flow, soil moisture, etc.
sdate: string or datetime object
start date of the data
edate: string or datetime object
end date of the data
Returns
-------
None
Usage
-----
>>> from pydsstools.utils import set_systemkey, contrail2dss
>>> set_systemkey('Valid System Key obtained from OneRain')
>>> contrail2dss(output='dss_out.dss',site_id=200,sensor_class=10,sdate='2000-01-01')
"""
output = kwargs['output']
site_id = kwargs['site_id']
sensor_class = kwargs['sensor_class']
close_dss = False
if isinstance(output,HecDss.Open):
fidout = output
else:
if not output[-4::].lower() != ".dss":
output = output + ".dss"
fidout = HecDss.Open(output)
close_dss = True
apart = kwargs.get('apart','')
bpart = kwargs.get('bpart','')
cpart = ' '.join(CONTRAIL_SENSOR_CLASSES[sensor_class].split()[0:-1])
dpart = kwargs.get('dpart','')
epart = 'IR-MONTH'
fpart = 'OneRain Site ID %s'%(site_id)
pathname = '/%s/%s/%s/%s/%s/%s/'%(apart,bpart,cpart,dpart,epart,fpart)
unit = CONTRAIL_SENSOR_CLASSES[sensor_class].split()[-1][1:-1]
data_type = 'INST'
if sensor_class == 11:
data_type = 'PER-CUM'
try:
fidout.deletePathname(pathname)
tsc = TimeSeriesContainer()
tsc.pathname = pathname
tsc.type = data_type
tsc.units = unit
tsc.interval = -1
time_values = get_contrail_timeseries(**kwargs)
if not time_values[0]:
logging.warn('Empty data retrieved from Contrail')
return
times = [HecTime(x).python_datetime for x in time_values[0]]
tsc.times = times
tsc.values = time_values[1]
tsc.numberValues = len(tsc.times)
fidout.put_ts(tsc)
except Exception as e:
raise e
finally:
if close_dss:
fidout.close()
def zstat2dss(zs_list, basin, ds, dss_file):
dates = [i.grid.date for i in zs_list]
sbasin_avg = [np.round(i.sbasin_avg, 4) for i in zs_list]
sbasin_vol = [i.sbasin_vol for i in zs_list]
basin_avg = [np.round(i.basin_avg, 4) for i in zs_list]
basin_vol = [i.basin_vol for i in zs_list]
names = [i.sbasin_names for i in zs_list]
date_rav = np.ravel(np.repeat(dates, len(zs_list[0].sbasin_names)))
sbasin_avg_rav = np.ravel(sbasin_avg)
sbasin_vol_rav = np.ravel(sbasin_vol)
names_rav = np.ravel(names)
idx = pd.MultiIndex.from_tuples(zip(date_rav, names_rav),
names=['date', 'name'])
sbasin = pd.DataFrame(index=idx,
data={
'mean_swe': sbasin_avg_rav,
'vol': sbasin_vol_rav
})
sbasin = sbasin.sort_index(level=0)
idx = pd.MultiIndex.from_tuples(zip(
dates, np.ravel(np.repeat('Total_Basin', len(dates)))),
names=['date', 'name'])
tbasin = pd.DataFrame(index=idx,
data={
'mean_swe': basin_avg,
'vol': basin_vol
})
tbasin = tbasin.sort_index(level=0)
idx = pd.date_range(
sbasin.index.get_level_values(0).min(),
sbasin.index.get_level_values(0).max())
fid = HecDss.Open(dss_file, version=6)
fid.close()
for name, group in sbasin.groupby(level=1):
#group.loc[:, 'wy'] = np.where(group.index.get_level_values(0).month>9,group.index.get_level_values(0).year+1,group.index.get_level_values(0).year)
group.index = group.index.droplevel(1)
#group.index = group.index.sort_values()
group = group.reindex(idx, fill_value=-901.0)
group.index = group.index + pd.DateOffset(hours=12)
start_date = group.index.min().strftime('%d%b%Y %H:%M:%S')
print(start_date)
pname = '/{0}/{1}/AVG_SWE//1DAY/{2}/'.format(
basin,
name.upper().replace(' ', '_'), ds)
print(pname)
tsc = TimeSeriesContainer()
tsc.granularity = 60 #seconds i.e. minute granularity
tsc.numberValues = group.mean_swe.size
tsc.startDateTime = start_date
tsc.pathname = pname
tsc.units = "M"
tsc.type = "INST-VAL"
tsc.interval = 1
#must a +ve integer for regular time-series
#actual interval implied from E part of pathname
tsc.values = group.mean_swe.values
#values may be list,array, numpy array
fid = HecDss.Open(dss_file)
fid.deletePathname(tsc.pathname)
status = fid.put(tsc)
fid.close()
pname = '/{0}/{1}/VOL//1DAY/{2}/'.format(
basin,
name.upper().replace(' ', '_'), ds)
print(pname)
tsc = TimeSeriesContainer()
tsc.granularity = 60 #seconds i.e. minute granularity
tsc.numberValues = group.index.size
tsc.startDateTime = start_date
tsc.pathname = pname
tsc.units = "CUBIC_METERS"
tsc.type = "INST-VAL"
tsc.interval = 1
#must a +ve integer for regular time-series
#actual interval implied from E part of pathname
tsc.values = group.vol.values
#values may be list,array, numpy array
fid = HecDss.Open(dss_file)
fid.deletePathname(tsc.pathname)
status = fid.put(tsc)
fid.close()
for name, group in tbasin.groupby(level=1):
#group.loc[:, 'wy'] = np.where(group.index.get_level_values(0).month>9,group.index.get_level_values(0).year+1,group.index.get_level_values(0).year)
group.index = group.index.droplevel(1)
#group.index = group.index.sort_values()
group = group.reindex(idx, fill_value=-901.0)
group.index = group.index + pd.DateOffset(hours=12)
start_date = group.index.min().strftime('%d%b%Y %H:%M:%S')
print(start_date)
pname = '/{0}/{1}/AVG_SWE//1DAY/{2}/'.format(
basin,
name.upper().replace(' ', '_'), ds)
print(pname)
tsc = TimeSeriesContainer()
tsc.granularity = 60 #seconds i.e. minute granularity
tsc.numberValues = group.mean_swe.size
tsc.startDateTime = start_date
tsc.pathname = pname
tsc.units = "M"
tsc.type = "INST-VAL"
tsc.interval = 1
#must a +ve integer for regular time-series
#actual interval implied from E part of pathname
tsc.values = group.mean_swe.values
#values may be list,array, numpy array
fid = HecDss.Open(dss_file)
fid.deletePathname(tsc.pathname)
status = fid.put(tsc)
fid.close()
pname = '/{0}/{1}/VOL//1DAY/{2}/'.format(
basin,
name.upper().replace(' ', '_'), ds)
print(pname)
tsc = TimeSeriesContainer()
tsc.granularity = 60 #seconds i.e. minute granularity
tsc.numberValues = group.index.size
tsc.startDateTime = start_date
tsc.pathname = pname
tsc.units = "CUBIC_METERS"
tsc.type = "INST-VAL"
tsc.interval = 1
#must a +ve integer for regular time-series
#actual interval implied from E part of pathname
tsc.values = group.vol.values
#values may be list,array, numpy array
fid = HecDss.Open(dss_file)
fid.deletePathname(tsc.pathname)
status = fid.put(tsc)
fid.close()
return sbasin, tbasin
'''
Write regular time-series data to example.dss
'''
import numpy as np
from datetime import datetime
from pydsstools.heclib.dss import HecDss
from pydsstools.core import TimeSeriesContainer
dss_file = "example.dss"
tsc = TimeSeriesContainer()
tsc.granularity_value = 60 #seconds i.e. minute granularity
tsc.numberValues = 10
tsc.startDateTime=datetime.now().strftime('%d %b %Y %H:00')
tsc.pathname = "/REGULAR/TIMESERIES/FLOW//1HOUR/WRITE2/"
tsc.units = "cfs"
tsc.type = "INST"
tsc.interval = 1
#must a +ve integer for regular time-series
#actual interval implied from E part of pathname
tsc.values =np.array(range(10),dtype=np.float32)
#values may be list,array, numpy array
fid = HecDss.Open(dss_file)
fid.deletePathname(tsc.pathname)
status = fid.put(tsc)
fid.close()
'''
Write irregular time-series data
Notes:
The interval must be [any] integer <= 0 for irregular time-series.
DParts: IR-MONTH, IR-YEAR, IR-DECADE, IR-CENTURY
'''
from datetime import datetime
from pydsstools.heclib.dss import HecDss
from pydsstools.core import TimeSeriesContainer, UNDEFINED
dss_file = "example.dss"
pathname = "/IRREGULAR/TIMESERIES/FLOW//IR-DECADE/Ex3/"
tsc = TimeSeriesContainer()
tsc.numberValues = 5
tsc.pathname = pathname
tsc.units = "cfs"
tsc.type = "INST"
tsc.interval = -1
tsc.values = [100, UNDEFINED, 500, 5000, 10000]
tsc.times = [
datetime(1900, 1, 12),
datetime(1950, 6, 2, 12),
datetime(1999, 12, 31, 23, 0, 0),
datetime(2009, 1, 20),
datetime(2019, 7, 15, 5, 0)
]
def _csv_to_dss(csv_filename: str,
output_path: str = None,
sim_name: str = None,
start_date: str = "01JAN2001 24:00:00"):
"""
Converts csv file to equivalent file in hec format
Parameters
----------
csv_filename : str
Complete or relative path of csv filename
output_path : str, default None
Folder directory to ouput converted hec files
sim_name : str, default None
Alternative name of output file
start_date : str, default "01JAN2001 00:00:00"
Start date associated with first row of data in csv filename
Returns
-------
None
"""
if output_path is None:
output_path = os.path.dirname(csv_filename)
if sim_name is None:
sim_name = os.path.splitext(os.path.basename(csv_filename))[0]
if not os.path.isdir(output_path):
try:
os.makedirs(output_path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
dss_filename = os.path.join(output_path, sim_name + '.dss')
# copy empty.dss to dss_filename
shutil.copy2(
os.path.join(os.path.dirname(__file__), 'templates', 'empty.dss'),
dss_filename)
# Prepare time-series data
df = pd.read_csv(csv_filename)
if df.shape[0] > 0:
while df.shape[0] < 365:
df = df.append(df.iloc[-1, :])
df = df.reset_index().drop(columns=['index'])
df = df.round(decimals=2)
tsc = TimeSeriesContainer()
tsc.startDateTime = start_date
tsc.numberValues = df.shape[0]
tsc.units = "cms"
tsc.type = "INST-VAL"
tsc.interval = 24 * 60
fid = HecDss.Open(dss_filename)
# add each column time-series from dataframe to hec
for column in df:
pathname = "/{}/{}///1DAY/{}/".format(sim_name, column, sim_name)
tsc.pathname = pathname
tsc.values = df[column].values
fid.deletePathname(tsc.pathname)
fid.put_ts(tsc)
fid.close()
'''
Write irregular time-series data to example.dss
'''
from datetime import datetime, timedelta
from array import array
from random import randrange
from pydsstools.heclib.utils import HecTime
from pydsstools.heclib.dss import HecDss
from pydsstools.core import TimeSeriesContainer
dss_file = "example.dss"
tsc = TimeSeriesContainer()
tsc.granularity_value = 60 #second i.e. minute granularity
tsc.numberValues = 10
tsc.pathname = "/IRREGULAR/TIMESERIES///IR-CENTURY/WRITE/"
#IR-MONTH, IR-YEAR, IR-DECADE, IR-CENTURY
tsc.units = "cfs"
tsc.type = "INST"
tsc.interval = -1
#-1 for specifying irregular time-series
tsc.values = array("d", range(10))
#values may be list, python array, or numpy array
times = []
begin = datetime(1899, 12, 31)
end = datetime(2017, 4, 18)
for x in range(tsc.numberValues):
diff = end - begin
diff_seconds = diff.days * 24 * 60 * 60 + diff.seconds
_rand = randrange(diff_seconds)