Ejemplo n.º 1
0
def main():
    configfile = get_args()
    config = parse_config(configfile)
    if verbose:
        for section in config.iterkeys():
            print '{}'.format(section)
            for key, value in config[section].iteritems():
                print '{:18s}: {}'.format(key, value)
    inpath = config['OPTIONS']['inpath']
    if inpath[-1] != '/':
        inpath += '/'
    filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*")
    domainfile = config['OPTIONS']['domainfile']
    columns = {}
    columns['runoff'] = int(config['OPTIONS']['runoff_column']) - 1
    columns['baseflow'] = int(config['OPTIONS']['baseflow_column']) - 1
    period = {}
    period['start'] = parse_date(config['OPTIONS']['start'])
    period['end'] = parse_date(config['OPTIONS']['end'])
    period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval']))
    outfiletemplate = config['OPTIONS']['outfiletemplate']
    ncvarinfo = {}
    ncvarinfo['name'] = config['OPTIONS']['ncvar_name']
    ncvarinfo['long_name'] = config['OPTIONS']['ncvar_longname']
    ncvarinfo['units'] = config['OPTIONS']['ncvar_units']
    ncvarinfo['divideby'] = float(config['OPTIONS']['ncvar_divideby'])
    ncformat = config['OPTIONS']['ncformat']

    vicasc2routenc(filelist, domainfile, columns, period, outfiletemplate,
                   ncvarinfo, ncformat)

    return
Ejemplo n.º 2
0
def main():
    configfile = get_args()
    config = parse_config(configfile)
    if verbose:
        for section in config.iterkeys():
            print '{}'.format(section)
            for key, value in config[section].iteritems():
                print '{:18s}: {}'.format(key, value)
    inpath = config['OPTIONS']['inpath']
    if inpath[-1] != '/':
        inpath += '/'
    filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*")
    domainfile = config['OPTIONS']['domainfile']
    period = {}
    period['start'] = parse_date(config['OPTIONS']['start'])
    period['end'] = parse_date(config['OPTIONS']['end'])
    period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval']))
    ncfile = config['OPTIONS']['ncfile']
    ncformat = config['OPTIONS']['ncformat']
    ncvarinfo = {}
    varlist = filterPick(config['OPTIONS'].keys(), re.compile('^ncvar.*_name$'))
    varlist = [x.split('_name')[0] for x in varlist]
    for var in varlist:
        ncvarinfo[var] = {}
        ncvarinfo[var]['name'] = config['OPTIONS'][var + '_name']
        ncvarinfo[var]['longname'] = config['OPTIONS'][var + '_longname']
        ncvarinfo[var]['column'] = config['OPTIONS'][var + '_column']
        ncvarinfo[var]['units'] = config['OPTIONS'][var + '_units']
        ncvarinfo[var]['divideby'] = float(config['OPTIONS'][var + '_divideby'])

    vicasc2nc(filelist, domainfile, period, ncfile,
              ncvarinfo, ncformat)

    return
def parse_commandline():
    parser = argparse.ArgumentParser(description='Determine polynomial '
                                     'relating streamflow and soil '
                                     'moisture for VIC streamflow '
                                     'assimilation')
    parser.add_argument('--station', required=True,
                        metavar='<station>',
                        help='station')
    parser.add_argument('--config', required=True,
                        metavar='<configuration file>',
                        help='Configuration file')

    args = parser.parse_args()

    configparser = ConfigParser.ConfigParser()
    configparser.read(os.path.expanduser(args.config))
    config = {}
    for section in configparser.sections():
        for key, value in configparser.items(section):
            config[key] = value

    config['timeofconcentration'] = int(config['timeofconcentration'])
    config['startwindow'] = parse_date(config['startwindow'], '-')
    config['endwindow'] = parse_date(config['endwindow'], '-')
    if config['figuretemplate'] == 'None':
        config['figuretemplate'] = None

    return (args.station, config)
Ejemplo n.º 4
0
def main():
    configfile = get_args()
    config = parse_config(configfile)
    if verbose:
        for section in config.iterkeys():
            print '{}'.format(section)
            for key, value in config[section].iteritems():
                print '{:18s}: {}'.format(key, value)
    inpath = config['OPTIONS']['inpath']
    if inpath[-1] != '/':
        inpath += '/'
    filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*")
    domainfile = config['OPTIONS']['domainfile']
    period = {}
    period['start'] = parse_date(config['OPTIONS']['start'])
    period['end'] = parse_date(config['OPTIONS']['end'])
    period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval']))
    ncfile = config['OPTIONS']['ncfile']
    ncformat = config['OPTIONS']['ncformat']
    ncvarinfo = {}
    varlist = filterPick(config['OPTIONS'].keys(),
                         re.compile('^ncvar.*_name$'))
    varlist = [x.split('_name')[0] for x in varlist]
    for var in varlist:
        ncvarinfo[var] = {}
        ncvarinfo[var]['name'] = config['OPTIONS'][var + '_name']
        ncvarinfo[var]['longname'] = config['OPTIONS'][var + '_longname']
        ncvarinfo[var]['column'] = config['OPTIONS'][var + '_column']
        ncvarinfo[var]['units'] = config['OPTIONS'][var + '_units']
        ncvarinfo[var]['divideby'] = float(config['OPTIONS'][var +
                                                             '_divideby'])

    vicasc2nc(filelist, domainfile, period, ncfile, ncvarinfo, ncformat)

    return
def parse_commandline():
    parser = argparse.ArgumentParser(description='Determine polynomial '
                                     'relating streamflow and soil '
                                     'moisture for VIC streamflow '
                                     'assimilation')
    parser.add_argument('--station',
                        required=True,
                        metavar='<station>',
                        help='station')
    parser.add_argument('--config',
                        required=True,
                        metavar='<configuration file>',
                        help='Configuration file')

    args = parser.parse_args()

    configparser = ConfigParser.ConfigParser()
    configparser.read(os.path.expanduser(args.config))
    config = {}
    for section in configparser.sections():
        for key, value in configparser.items(section):
            config[key] = value

    config['timeofconcentration'] = int(config['timeofconcentration'])
    config['startwindow'] = parse_date(config['startwindow'], '-')
    config['endwindow'] = parse_date(config['endwindow'], '-')
    if config['figuretemplate'] == 'None':
        config['figuretemplate'] = None

    return (args.station, config)
Ejemplo n.º 6
0
def main():
    configfile = get_args()
    config = parse_config(configfile)
    if verbose:
        for section in config.iterkeys():
            print "{}".format(section)
            for key, value in config[section].iteritems():
                print "{:18s}: {}".format(key, value)
    inpath = config["OPTIONS"]["inpath"]
    if inpath[-1] != "/":
        inpath += "/"
    filelist = glob.glob(inpath + config["OPTIONS"]["infiletemplate"] + "*")
    domainfile = config["OPTIONS"]["domainfile"]
    columns = {}
    columns["runoff"] = int(config["OPTIONS"]["runoff_column"]) - 1
    columns["baseflow"] = int(config["OPTIONS"]["baseflow_column"]) - 1
    period = {}
    period["start"] = parse_date(config["OPTIONS"]["start"])
    period["end"] = parse_date(config["OPTIONS"]["end"])
    period["interval"] = dt.timedelta(int(config["OPTIONS"]["timeinterval"]))
    outfiletemplate = config["OPTIONS"]["outfiletemplate"]
    ncvarinfo = {}
    ncvarinfo["name"] = config["OPTIONS"]["ncvar_name"]
    ncvarinfo["long_name"] = config["OPTIONS"]["ncvar_longname"]
    ncvarinfo["units"] = config["OPTIONS"]["ncvar_units"]
    ncvarinfo["divideby"] = float(config["OPTIONS"]["ncvar_divideby"])
    ncformat = config["OPTIONS"]["ncformat"]

    vicasc2routenc(filelist, domainfile, columns, period, outfiletemplate, ncvarinfo, ncformat)

    return
def parse_commandline():
    parser = argparse.ArgumentParser(description='Update VIC state file '
                                     'based on assimilation of observed '
                                     'streamflow')
    parser.add_argument('--date', required=True,
                        metavar='<date>',
                        help='Forecast date: YYYY-MM-DD')
    parser.add_argument('--stations', required=True,
                        metavar='<stations>',
                        help='Comma-delimited list of stations to use '
                        'in the update')
    parser.add_argument('--config', required=True,
                        metavar='<configuration file>',
                        help='Configuration file')

    args = parser.parse_args()

    forecastdate = parse_date(args.date, '-')

    configparser = ConfigParser.ConfigParser()
    configparser.read(os.path.expanduser(args.config))
    config = {}
    for section in configparser.sections():
        for key, value in configparser.items(section):
            config[key] = value

    config['timeofconcentration'] = int(config['timeofconcentration'])
    config['latlondigits'] = int(config['latlondigits'])

    stations = dict((station, config['timeofconcentration'])
                    for station in args.stations.split(','))

    return (forecastdate, stations, config)
def parse_commandline():
    parser = argparse.ArgumentParser(description='Update VIC state file '
                                     'based on assimilation of observed '
                                     'streamflow')
    parser.add_argument('--date',
                        required=True,
                        metavar='<date>',
                        help='Forecast date: YYYY-MM-DD')
    parser.add_argument('--stations',
                        required=True,
                        metavar='<stations>',
                        help='Comma-delimited list of stations to use '
                        'in the update')
    parser.add_argument('--config',
                        required=True,
                        metavar='<configuration file>',
                        help='Configuration file')

    args = parser.parse_args()

    forecastdate = parse_date(args.date, '-')

    configparser = ConfigParser.ConfigParser()
    configparser.read(os.path.expanduser(args.config))
    config = {}
    for section in configparser.sections():
        for key, value in configparser.items(section):
            config[key] = value

    config['timeofconcentration'] = int(config['timeofconcentration'])
    config['latlondigits'] = int(config['latlondigits'])

    stations = dict((station, config['timeofconcentration'])
                    for station in args.stations.split(','))

    return (forecastdate, stations, config)
import os
import glob
import pandas as pd
from utils.parse import parse_date

if len(sys.argv) < 6:
    print 'Must provide inpath, outpath, filemask, startdate and enddate'
    sys.exit()

inpath = sys.argv[1]
outpath = sys.argv[2]
filemask = sys.argv[3]
startdate = sys.argv[4]
enddate = sys.argv[5]

start = parse_date(startdate, '/')
end = parse_date(enddate, '/')
dates = pd.date_range(start, end, freq='D')

filelist = glob.glob('{}/{}*'.format(inpath, filemask))

# all the files are the same, so we need to determine the indices only once
infile = filelist[0]
data = np.loadtxt(infile,
                  dtype={'names': ('prcp', 'tmax', 'tmin', 'wind'),
                         'formats': ('f4', 'f4', 'f4', 'f4')})
# determine the number of months
nyears = end.year - start.year + 1

# initialize empty arrays
idx = np.empty((data['prcp'].size, nyears), dtype='bool')*False
Ejemplo n.º 10
0
    def __init__(self, configparser, configurationfile):
        """Initialize a particle filter based on a configurationfile"""
        self.pool = None
        # Parse the configuration file
        self.configfile = configurationfile
        self.config = collections.OrderedDict()  # preserve order of entries
        for section in configparser.sections():
            self.config[section] = collections.OrderedDict()
            for key, value in configparser.items(section):
                self.config[section][key] = value

        if not self.config['COMPUTING']['poolsize']:
            self.poolsize = multiprocessing.cpu_count()
        else:
            self.poolsize = configparser.getint('COMPUTING', 'poolsize')

        self.ID = self.config['FILTER']['runid']
        logging.info('Initializing particle filter: {}'.format(self.ID))

        ncvarinfo = {}
        varlist = filterPick(self.config['VICASC2NC'].keys(),
                             re.compile('^ncvar.*_name$'))
        varlist = [x.split('_name')[0] for x in varlist]
        for var in varlist:
            ncvarinfo[var] = {}
            ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name']
            ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var +
                                                                  '_longname']
            ncvarinfo[var]['column'] = self.config['VICASC2NC'][var +
                                                                '_column']
            ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units']
            ncvarinfo[var]['divideby'] = float(
                self.config['VICASC2NC'][var + '_divideby'])
        self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo

        # initialize simulation time
        self.start = parse_date(self.config['SIMULATION']['start'])
        self.now = self.start
        self.end = parse_date(self.config['SIMULATION']['end'])
        self.runlength = dt.timedelta(
            configparser.getint('SIMULATION', 'runlength'))
        self.backstep = dt.timedelta(
            configparser.getint('SIMULATION', 'backstep'))

        # number of timestep to evaluate
        self.neval = configparser.getint('FILTER', 'neval')

        # Initialize particles
        self.np = configparser.getint('FILTER', 'np')
        self.particles = [
            VicParticle(i + 1, 1. / self.np) for i in range(self.np)
        ]
        for i in range(self.np):
            p = self.particles[i]
            p.vic['exe'] = self.config['MODELS']['vicexe']
            p.route['exe'] = self.config['MODELS']['routeexe']
            p.refvar = self.config['FILTER']['referencevar']
            p.initstatefile = {}
            p.initstatefile['vic'] = self.config['VICCONFIG'][
                'INIT_STATE'].format(p.ID)
            if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower(
            ) == 'startup':
                p.initstatefile['route'] = \
                    self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\
                    format(p.ID)
            p.neval = self.neval
        self.normalize_weights()
        logging.info('Initialized {} particles'.format(self.np))

        # set up run and archive paths for particles
        runpath = self.config['FILTER']['rundir']
        archivepath = self.config['FILTER']['archivedir']
        subdirs = ['log', 'history', 'settings']
        self.paths = {}
        self.paths['run'] = {}
        self.paths['archive'] = {}
        for subdir in subdirs:
            self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
            self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
        logging.debug('Creating paths for filter {}:'.format(self.ID))
        for topdir, dirdict in self.paths.iteritems():
            logging.debug('\t{:8s}:'.format(topdir))
            for key, val in dirdict.iteritems():
                logging.debug('\t\t{:6s}: {}'.format(key, val))
                make_sure_path_exists(val)

        subdirs = ['config', 'data', 'log', 'state', 'history']
        for p in self.particles:
            p.paths = {}
            p.paths['run'] = {}
            p.paths['archive'] = {}
            runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'],
                                             p.ID)
            archivepath = '{}/pid_{:03d}'.format(
                self.config['FILTER']['archivedir'], p.ID)
            for subdir in subdirs:
                p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
                p.paths['archive'][subdir] = '{}/{}'.format(
                    archivepath, subdir)
            logging.debug('Creating paths for particle {:03d}:'.format(p.ID))
            for topdir, dirdict in p.paths.iteritems():
                logging.debug('\t{:8s}:'.format(topdir))
                for key, val in dirdict.iteritems():
                    logging.debug('\t\t{:6s}: {}'.format(key, val))
                    make_sure_path_exists(val)

        # set history file for filter
        filename = '{}.history'.format(self.ID)
        self.hist = open(os.path.join(self.paths['run']['history'], filename),
                         'w')

        # set history file for each particle
        for p in self.particles:
            filename = '{}.{:03d}.history'.format(self.ID, p.ID)
            p.hist = open(os.path.join(p.paths['run']['history'], filename),
                          'w')

        logging.info('Initialization complete')
Ejemplo n.º 11
0
    def __init__(self, configparser, configurationfile):
        """Initialize a particle filter based on a configurationfile"""
        self.pool = None
        # Parse the configuration file
        self.configfile = configurationfile
        self.config = collections.OrderedDict() # preserve order of entries
        for section in configparser.sections():
            self.config[section] = collections.OrderedDict()
            for key, value in configparser.items(section):
                self.config[section][key] = value

        if not self.config['COMPUTING']['poolsize']:
            self.poolsize = multiprocessing.cpu_count()
        else:
            self.poolsize = configparser.getint('COMPUTING','poolsize')

        self.ID = self.config['FILTER']['runid']
        logging.info('Initializing particle filter: {}'.format(self.ID))

        ncvarinfo = {}
        varlist = filterPick(self.config['VICASC2NC'].keys(), re.compile('^ncvar.*_name$'))
        varlist = [x.split('_name')[0] for x in varlist]
        for var in varlist:
            ncvarinfo[var] = {}
            ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name']
            ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var + '_longname']
            ncvarinfo[var]['column'] = self.config['VICASC2NC'][var + '_column']
            ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units']
            ncvarinfo[var]['divideby'] = float(self.config['VICASC2NC'][var + '_divideby'])
        self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo

        # initialize simulation time
        self.start = parse_date(self.config['SIMULATION']['start'])
        self.now = self.start
        self.end = parse_date(self.config['SIMULATION']['end'])
        self.runlength = dt.timedelta(configparser.getint('SIMULATION',
                                                          'runlength'))
        self.backstep = dt.timedelta(configparser.getint('SIMULATION',
                                                         'backstep'))

        # number of timestep to evaluate
        self.neval = configparser.getint('FILTER','neval')

        # Initialize particles
        self.np = configparser.getint('FILTER', 'np')
        self.particles = [VicParticle(i+1, 1./self.np) for i in range(self.np)]
        for i in range(self.np):
            p = self.particles[i]
            p.vic['exe'] = self.config['MODELS']['vicexe']
            p.route['exe'] = self.config['MODELS']['routeexe']
            p.refvar = self.config['FILTER']['referencevar']
            p.initstatefile = {}
            p.initstatefile['vic'] = self.config['VICCONFIG']['INIT_STATE'].format(p.ID)
            if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower() == 'startup':
                p.initstatefile['route'] = \
                    self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\
                    format(p.ID)
            p.neval = self.neval
        self.normalize_weights()
        logging.info('Initialized {} particles'.format(self.np))

        # set up run and archive paths for particles
        runpath = self.config['FILTER']['rundir']
        archivepath = self.config['FILTER']['archivedir']
        subdirs = ['log', 'history', 'settings']
        self.paths = {}
        self.paths['run'] = {}
        self.paths['archive'] = {}
        for subdir in subdirs:
            self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
            self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
        logging.debug('Creating paths for filter {}:'.format(self.ID))
        for topdir, dirdict in self.paths.iteritems():
            logging.debug('\t{:8s}:'.format(topdir))
            for key, val in dirdict.iteritems():
                logging.debug('\t\t{:6s}: {}'.format(key, val))
                make_sure_path_exists(val)

        subdirs = ['config', 'data', 'log', 'state', 'history']
        for p in self.particles:
            p.paths = {}
            p.paths['run'] = {}
            p.paths['archive'] = {}
            runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'], p.ID)
            archivepath = '{}/pid_{:03d}'.format(self.config['FILTER']['archivedir'], p.ID)
            for subdir in subdirs:
                p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir)
                p.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir)
            logging.debug('Creating paths for particle {:03d}:'.format(p.ID))
            for topdir, dirdict in p.paths.iteritems():
                logging.debug('\t{:8s}:'.format(topdir))
                for key, val in dirdict.iteritems():
                    logging.debug('\t\t{:6s}: {}'.format(key, val))
                    make_sure_path_exists(val)

        # set history file for filter
        filename = '{}.history'.format(self.ID)
        self.hist = open(os.path.join(self.paths['run']['history'], filename), 'w')

        # set history file for each particle
        for p in self.particles:
            filename = '{}.{:03d}.history'.format(self.ID, p.ID)
            p.hist = open(os.path.join(p.paths['run']['history'], filename), 'w')

        logging.info('Initialization complete')