def main():
configfile = get_args()
config = parse_config(configfile)
if verbose:
for section in config.iterkeys():
print '{}'.format(section)
for key, value in config[section].iteritems():
print '{:18s}: {}'.format(key, value)
inpath = config['OPTIONS']['inpath']
if inpath[-1] != '/':
inpath += '/'
filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*")
domainfile = config['OPTIONS']['domainfile']
columns = {}
columns['runoff'] = int(config['OPTIONS']['runoff_column']) - 1
columns['baseflow'] = int(config['OPTIONS']['baseflow_column']) - 1
period = {}
period['start'] = parse_date(config['OPTIONS']['start'])
period['end'] = parse_date(config['OPTIONS']['end'])
period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval']))
outfiletemplate = config['OPTIONS']['outfiletemplate']
ncvarinfo = {}
ncvarinfo['name'] = config['OPTIONS']['ncvar_name']
ncvarinfo['long_name'] = config['OPTIONS']['ncvar_longname']
ncvarinfo['units'] = config['OPTIONS']['ncvar_units']
ncvarinfo['divideby'] = float(config['OPTIONS']['ncvar_divideby'])
ncformat = config['OPTIONS']['ncformat']
vicasc2routenc(filelist, domainfile, columns, period, outfiletemplate,
ncvarinfo, ncformat)
return
def main():
configfile = get_args()
config = parse_config(configfile)
if verbose:
for section in config.iterkeys():
print '{}'.format(section)
for key, value in config[section].iteritems():
print '{:18s}: {}'.format(key, value)
inpath = config['OPTIONS']['inpath']
if inpath[-1] != '/':
inpath += '/'
filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*")
domainfile = config['OPTIONS']['domainfile']
period = {}
period['start'] = parse_date(config['OPTIONS']['start'])
period['end'] = parse_date(config['OPTIONS']['end'])
period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval']))
ncfile = config['OPTIONS']['ncfile']
ncformat = config['OPTIONS']['ncformat']
ncvarinfo = {}
varlist = filterPick(config['OPTIONS'].keys(), re.compile('^ncvar.*_name$'))
varlist = [x.split('_name')[0] for x in varlist]
for var in varlist:
ncvarinfo[var] = {}
ncvarinfo[var]['name'] = config['OPTIONS'][var + '_name']
ncvarinfo[var]['longname'] = config['OPTIONS'][var + '_longname']
ncvarinfo[var]['column'] = config['OPTIONS'][var + '_column']
ncvarinfo[var]['units'] = config['OPTIONS'][var + '_units']
ncvarinfo[var]['divideby'] = float(config['OPTIONS'][var + '_divideby'])
vicasc2nc(filelist, domainfile, period, ncfile,
ncvarinfo, ncformat)
return
def parse_commandline():
parser = argparse.ArgumentParser(description='Determine polynomial '
'relating streamflow and soil '
'moisture for VIC streamflow '
'assimilation')
parser.add_argument('--station', required=True,
metavar='<station>',
help='station')
parser.add_argument('--config', required=True,
metavar='<configuration file>',
help='Configuration file')
args = parser.parse_args()
configparser = ConfigParser.ConfigParser()
configparser.read(os.path.expanduser(args.config))
config = {}
for section in configparser.sections():
for key, value in configparser.items(section):
config[key] = value
config['timeofconcentration'] = int(config['timeofconcentration'])
config['startwindow'] = parse_date(config['startwindow'], '-')
config['endwindow'] = parse_date(config['endwindow'], '-')
if config['figuretemplate'] == 'None':
config['figuretemplate'] = None
return (args.station, config)
def main():
configfile = get_args()
config = parse_config(configfile)
if verbose:
for section in config.iterkeys():
print '{}'.format(section)
for key, value in config[section].iteritems():
print '{:18s}: {}'.format(key, value)
inpath = config['OPTIONS']['inpath']
if inpath[-1] != '/':
inpath += '/'
filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*")
domainfile = config['OPTIONS']['domainfile']
period = {}
period['start'] = parse_date(config['OPTIONS']['start'])
period['end'] = parse_date(config['OPTIONS']['end'])
period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval']))
ncfile = config['OPTIONS']['ncfile']
ncformat = config['OPTIONS']['ncformat']
ncvarinfo = {}
varlist = filterPick(config['OPTIONS'].keys(),
re.compile('^ncvar.*_name$'))
varlist = [x.split('_name')[0] for x in varlist]
for var in varlist:
ncvarinfo[var] = {}
ncvarinfo[var]['name'] = config['OPTIONS'][var + '_name']
ncvarinfo[var]['longname'] = config['OPTIONS'][var + '_longname']
ncvarinfo[var]['column'] = config['OPTIONS'][var + '_column']
ncvarinfo[var]['units'] = config['OPTIONS'][var + '_units']
ncvarinfo[var]['divideby'] = float(config['OPTIONS'][var +
'_divideby'])
vicasc2nc(filelist, domainfile, period, ncfile, ncvarinfo, ncformat)
return
def parse_commandline():
parser = argparse.ArgumentParser(description='Determine polynomial '
'relating streamflow and soil '
'moisture for VIC streamflow '
'assimilation')
parser.add_argument('--station',
required=True,
metavar='<station>',
help='station')
parser.add_argument('--config',
required=True,
metavar='<configuration file>',
help='Configuration file')
args = parser.parse_args()
configparser = ConfigParser.ConfigParser()
configparser.read(os.path.expanduser(args.config))
config = {}
for section in configparser.sections():
for key, value in configparser.items(section):
config[key] = value
config['timeofconcentration'] = int(config['timeofconcentration'])
config['startwindow'] = parse_date(config['startwindow'], '-')
config['endwindow'] = parse_date(config['endwindow'], '-')
if config['figuretemplate'] == 'None':
config['figuretemplate'] = None
return (args.station, config)
def main():
configfile = get_args()
config = parse_config(configfile)
if verbose:
for section in config.iterkeys():
print "{}".format(section)
for key, value in config[section].iteritems():
print "{:18s}: {}".format(key, value)
inpath = config["OPTIONS"]["inpath"]
if inpath[-1] != "/":
inpath += "/"
filelist = glob.glob(inpath + config["OPTIONS"]["infiletemplate"] + "*")
domainfile = config["OPTIONS"]["domainfile"]
columns = {}
columns["runoff"] = int(config["OPTIONS"]["runoff_column"]) - 1
columns["baseflow"] = int(config["OPTIONS"]["baseflow_column"]) - 1
period = {}
period["start"] = parse_date(config["OPTIONS"]["start"])
period["end"] = parse_date(config["OPTIONS"]["end"])
period["interval"] = dt.timedelta(int(config["OPTIONS"]["timeinterval"]))
outfiletemplate = config["OPTIONS"]["outfiletemplate"]
ncvarinfo = {}
ncvarinfo["name"] = config["OPTIONS"]["ncvar_name"]
ncvarinfo["long_name"] = config["OPTIONS"]["ncvar_longname"]
ncvarinfo["units"] = config["OPTIONS"]["ncvar_units"]
ncvarinfo["divideby"] = float(config["OPTIONS"]["ncvar_divideby"])
ncformat = config["OPTIONS"]["ncformat"]
vicasc2routenc(filelist, domainfile, columns, period, outfiletemplate, ncvarinfo, ncformat)
return
def parse_commandline():
parser = argparse.ArgumentParser(description='Update VIC state file '
'based on assimilation of observed '
'streamflow')
parser.add_argument('--date', required=True,
metavar='<date>',
help='Forecast date: YYYY-MM-DD')
parser.add_argument('--stations', required=True,
metavar='<stations>',
help='Comma-delimited list of stations to use '
'in the update')
parser.add_argument('--config', required=True,
metavar='<configuration file>',
help='Configuration file')
args = parser.parse_args()
forecastdate = parse_date(args.date, '-')
configparser = ConfigParser.ConfigParser()
configparser.read(os.path.expanduser(args.config))
config = {}
for section in configparser.sections():
for key, value in configparser.items(section):
config[key] = value
config['timeofconcentration'] = int(config['timeofconcentration'])
config['latlondigits'] = int(config['latlondigits'])
stations = dict((station, config['timeofconcentration'])
for station in args.stations.split(','))
return (forecastdate, stations, config)
def parse_commandline():
parser = argparse.ArgumentParser(description='Update VIC state file '
'based on assimilation of observed '
'streamflow')
parser.add_argument('--date',
required=True,
metavar='<date>',
help='Forecast date: YYYY-MM-DD')
parser.add_argument('--stations',
required=True,
metavar='<stations>',
help='Comma-delimited list of stations to use '
'in the update')
parser.add_argument('--config',
required=True,
metavar='<configuration file>',
help='Configuration file')
args = parser.parse_args()
forecastdate = parse_date(args.date, '-')
configparser = ConfigParser.ConfigParser()
configparser.read(os.path.expanduser(args.config))
config = {}
for section in configparser.sections():
for key, value in configparser.items(section):
config[key] = value
config['timeofconcentration'] = int(config['timeofconcentration'])
config['latlondigits'] = int(config['latlondigits'])
stations = dict((station, config['timeofconcentration'])
for station in args.stations.split(','))
return (forecastdate, stations, config)
import os
import glob
import pandas as pd
from utils.parse import parse_date
if len(sys.argv) < 6:
print 'Must provide inpath, outpath, filemask, startdate and enddate'
sys.exit()
inpath = sys.argv[1]
outpath = sys.argv[2]
filemask = sys.argv[3]
startdate = sys.argv[4]
enddate = sys.argv[5]
start = parse_date(startdate, '/')
end = parse_date(enddate, '/')
dates = pd.date_range(start, end, freq='D')
filelist = glob.glob('{}/{}*'.format(inpath, filemask))
# all the files are the same, so we need to determine the indices only once
infile = filelist[0]
data = np.loadtxt(infile,
dtype={'names': ('prcp', 'tmax', 'tmin', 'wind'),
'formats': ('f4', 'f4', 'f4', 'f4')})
# determine the number of months
nyears = end.year - start.year + 1
# initialize empty arrays
idx = np.empty((data['prcp'].size, nyears), dtype='bool')*False
def __init__(self, configparser, configurationfile):
"""Initialize a particle filter based on a configurationfile"""
self.pool = None
# Parse the configuration file
self.configfile = configurationfile
self.config = collections.OrderedDict() # preserve order of entries
for section in configparser.sections():
self.config[section] = collections.OrderedDict()
for key, value in configparser.items(section):
self.config[section][key] = value
if not self.config['COMPUTING']['poolsize']:
self.poolsize = multiprocessing.cpu_count()
else:
self.poolsize = configparser.getint('COMPUTING', 'poolsize')
self.ID = self.config['FILTER']['runid']
logging.info('Initializing particle filter: {}'.format(self.ID))
ncvarinfo = {}
varlist = filterPick(self.config['VICASC2NC'].keys(),
re.compile('^ncvar.*_name$'))
varlist = [x.split('_name')[0] for x in varlist]
for var in varlist:
ncvarinfo[var] = {}
ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name']
ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var +
'_longname']
ncvarinfo[var]['column'] = self.config['VICASC2NC'][var +
'_column']
ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units']
ncvarinfo[var]['divideby'] = float(
self.config['VICASC2NC'][var + '_divideby'])
self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo
# initialize simulation time
self.start = parse_date(self.config['SIMULATION']['start'])
self.now = self.start
self.end = parse_date(self.config['SIMULATION']['end'])
self.runlength = dt.timedelta(
configparser.getint('SIMULATION', 'runlength'))
self.backstep = dt.timedelta(
configparser.getint('SIMULATION', 'backstep'))
# number of timestep to evaluate
self.neval = configparser.getint('FILTER', 'neval')
# Initialize particles
self.np = configparser.getint('FILTER', 'np')
self.particles = [
VicParticle(i + 1, 1. / self.np) for i in range(self.np)
]
for i in range(self.np):
p = self.particles[i]
p.vic['exe'] = self.config['MODELS']['vicexe']
p.route['exe'] = self.config['MODELS']['routeexe']
p.refvar = self.config['FILTER']['referencevar']
p.initstatefile = {}
p.initstatefile['vic'] = self.config['VICCONFIG'][
'INIT_STATE'].format(p.ID)
if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower(
) == 'startup':
p.initstatefile['route'] = \
self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\
format(p.ID)
p.neval = self.neval
self.normalize_weights()
logging.info('Initialized {} particles'.format(self.np))
# set up run and archive paths for particles
runpath = self.config['FILTER']['rundir']
archivepath = self.config['FILTER']['archivedir']
subdirs = ['log', 'history', 'settings']
self.paths = {}
self.paths['run'] = {}
self.paths['archive'] = {}
for subdir in subdirs:
self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
logging.debug('Creating paths for filter {}:'.format(self.ID))
for topdir, dirdict in self.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
subdirs = ['config', 'data', 'log', 'state', 'history']
for p in self.particles:
p.paths = {}
p.paths['run'] = {}
p.paths['archive'] = {}
runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'],
p.ID)
archivepath = '{}/pid_{:03d}'.format(
self.config['FILTER']['archivedir'], p.ID)
for subdir in subdirs:
p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
p.paths['archive'][subdir] = '{}/{}'.format(
archivepath, subdir)
logging.debug('Creating paths for particle {:03d}:'.format(p.ID))
for topdir, dirdict in p.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
# set history file for filter
filename = '{}.history'.format(self.ID)
self.hist = open(os.path.join(self.paths['run']['history'], filename),
'w')
# set history file for each particle
for p in self.particles:
filename = '{}.{:03d}.history'.format(self.ID, p.ID)
p.hist = open(os.path.join(p.paths['run']['history'], filename),
'w')
logging.info('Initialization complete')
def __init__(self, configparser, configurationfile):
"""Initialize a particle filter based on a configurationfile"""
self.pool = None
# Parse the configuration file
self.configfile = configurationfile
self.config = collections.OrderedDict() # preserve order of entries
for section in configparser.sections():
self.config[section] = collections.OrderedDict()
for key, value in configparser.items(section):
self.config[section][key] = value
if not self.config['COMPUTING']['poolsize']:
self.poolsize = multiprocessing.cpu_count()
else:
self.poolsize = configparser.getint('COMPUTING','poolsize')
self.ID = self.config['FILTER']['runid']
logging.info('Initializing particle filter: {}'.format(self.ID))
ncvarinfo = {}
varlist = filterPick(self.config['VICASC2NC'].keys(), re.compile('^ncvar.*_name$'))
varlist = [x.split('_name')[0] for x in varlist]
for var in varlist:
ncvarinfo[var] = {}
ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name']
ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var + '_longname']
ncvarinfo[var]['column'] = self.config['VICASC2NC'][var + '_column']
ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units']
ncvarinfo[var]['divideby'] = float(self.config['VICASC2NC'][var + '_divideby'])
self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo
# initialize simulation time
self.start = parse_date(self.config['SIMULATION']['start'])
self.now = self.start
self.end = parse_date(self.config['SIMULATION']['end'])
self.runlength = dt.timedelta(configparser.getint('SIMULATION',
'runlength'))
self.backstep = dt.timedelta(configparser.getint('SIMULATION',
'backstep'))
# number of timestep to evaluate
self.neval = configparser.getint('FILTER','neval')
# Initialize particles
self.np = configparser.getint('FILTER', 'np')
self.particles = [VicParticle(i+1, 1./self.np) for i in range(self.np)]
for i in range(self.np):
p = self.particles[i]
p.vic['exe'] = self.config['MODELS']['vicexe']
p.route['exe'] = self.config['MODELS']['routeexe']
p.refvar = self.config['FILTER']['referencevar']
p.initstatefile = {}
p.initstatefile['vic'] = self.config['VICCONFIG']['INIT_STATE'].format(p.ID)
if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower() == 'startup':
p.initstatefile['route'] = \
self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\
format(p.ID)
p.neval = self.neval
self.normalize_weights()
logging.info('Initialized {} particles'.format(self.np))
# set up run and archive paths for particles
runpath = self.config['FILTER']['rundir']
archivepath = self.config['FILTER']['archivedir']
subdirs = ['log', 'history', 'settings']
self.paths = {}
self.paths['run'] = {}
self.paths['archive'] = {}
for subdir in subdirs:
self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
logging.debug('Creating paths for filter {}:'.format(self.ID))
for topdir, dirdict in self.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
subdirs = ['config', 'data', 'log', 'state', 'history']
for p in self.particles:
p.paths = {}
p.paths['run'] = {}
p.paths['archive'] = {}
runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'], p.ID)
archivepath = '{}/pid_{:03d}'.format(self.config['FILTER']['archivedir'], p.ID)
for subdir in subdirs:
p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
p.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
logging.debug('Creating paths for particle {:03d}:'.format(p.ID))
for topdir, dirdict in p.paths.iteritems():
logging.debug('\t{:8s}:'.format(topdir))
for key, val in dirdict.iteritems():
logging.debug('\t\t{:6s}: {}'.format(key, val))
make_sure_path_exists(val)
# set history file for filter
filename = '{}.history'.format(self.ID)
self.hist = open(os.path.join(self.paths['run']['history'], filename), 'w')
# set history file for each particle
for p in self.particles:
filename = '{}.{:03d}.history'.format(self.ID, p.ID)
p.hist = open(os.path.join(p.paths['run']['history'], filename), 'w')
logging.info('Initialization complete')