def main(): configfile = get_args() config = parse_config(configfile) if verbose: for section in config.iterkeys(): print '{}'.format(section) for key, value in config[section].iteritems(): print '{:18s}: {}'.format(key, value) inpath = config['OPTIONS']['inpath'] if inpath[-1] != '/': inpath += '/' filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*") domainfile = config['OPTIONS']['domainfile'] columns = {} columns['runoff'] = int(config['OPTIONS']['runoff_column']) - 1 columns['baseflow'] = int(config['OPTIONS']['baseflow_column']) - 1 period = {} period['start'] = parse_date(config['OPTIONS']['start']) period['end'] = parse_date(config['OPTIONS']['end']) period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval'])) outfiletemplate = config['OPTIONS']['outfiletemplate'] ncvarinfo = {} ncvarinfo['name'] = config['OPTIONS']['ncvar_name'] ncvarinfo['long_name'] = config['OPTIONS']['ncvar_longname'] ncvarinfo['units'] = config['OPTIONS']['ncvar_units'] ncvarinfo['divideby'] = float(config['OPTIONS']['ncvar_divideby']) ncformat = config['OPTIONS']['ncformat'] vicasc2routenc(filelist, domainfile, columns, period, outfiletemplate, ncvarinfo, ncformat) return
def main(): configfile = get_args() config = parse_config(configfile) if verbose: for section in config.iterkeys(): print '{}'.format(section) for key, value in config[section].iteritems(): print '{:18s}: {}'.format(key, value) inpath = config['OPTIONS']['inpath'] if inpath[-1] != '/': inpath += '/' filelist = glob.glob(inpath + config['OPTIONS']['infiletemplate'] + "*") domainfile = config['OPTIONS']['domainfile'] period = {} period['start'] = parse_date(config['OPTIONS']['start']) period['end'] = parse_date(config['OPTIONS']['end']) period['interval'] = dt.timedelta(int(config['OPTIONS']['timeinterval'])) ncfile = config['OPTIONS']['ncfile'] ncformat = config['OPTIONS']['ncformat'] ncvarinfo = {} varlist = filterPick(config['OPTIONS'].keys(), re.compile('^ncvar.*_name$')) varlist = [x.split('_name')[0] for x in varlist] for var in varlist: ncvarinfo[var] = {} ncvarinfo[var]['name'] = config['OPTIONS'][var + '_name'] ncvarinfo[var]['longname'] = config['OPTIONS'][var + '_longname'] ncvarinfo[var]['column'] = config['OPTIONS'][var + '_column'] ncvarinfo[var]['units'] = config['OPTIONS'][var + '_units'] ncvarinfo[var]['divideby'] = float(config['OPTIONS'][var + '_divideby']) vicasc2nc(filelist, domainfile, period, ncfile, ncvarinfo, ncformat) return
def parse_commandline(): parser = argparse.ArgumentParser(description='Determine polynomial ' 'relating streamflow and soil ' 'moisture for VIC streamflow ' 'assimilation') parser.add_argument('--station', required=True, metavar='<station>', help='station') parser.add_argument('--config', required=True, metavar='<configuration file>', help='Configuration file') args = parser.parse_args() configparser = ConfigParser.ConfigParser() configparser.read(os.path.expanduser(args.config)) config = {} for section in configparser.sections(): for key, value in configparser.items(section): config[key] = value config['timeofconcentration'] = int(config['timeofconcentration']) config['startwindow'] = parse_date(config['startwindow'], '-') config['endwindow'] = parse_date(config['endwindow'], '-') if config['figuretemplate'] == 'None': config['figuretemplate'] = None return (args.station, config)
def main(): configfile = get_args() config = parse_config(configfile) if verbose: for section in config.iterkeys(): print "{}".format(section) for key, value in config[section].iteritems(): print "{:18s}: {}".format(key, value) inpath = config["OPTIONS"]["inpath"] if inpath[-1] != "/": inpath += "/" filelist = glob.glob(inpath + config["OPTIONS"]["infiletemplate"] + "*") domainfile = config["OPTIONS"]["domainfile"] columns = {} columns["runoff"] = int(config["OPTIONS"]["runoff_column"]) - 1 columns["baseflow"] = int(config["OPTIONS"]["baseflow_column"]) - 1 period = {} period["start"] = parse_date(config["OPTIONS"]["start"]) period["end"] = parse_date(config["OPTIONS"]["end"]) period["interval"] = dt.timedelta(int(config["OPTIONS"]["timeinterval"])) outfiletemplate = config["OPTIONS"]["outfiletemplate"] ncvarinfo = {} ncvarinfo["name"] = config["OPTIONS"]["ncvar_name"] ncvarinfo["long_name"] = config["OPTIONS"]["ncvar_longname"] ncvarinfo["units"] = config["OPTIONS"]["ncvar_units"] ncvarinfo["divideby"] = float(config["OPTIONS"]["ncvar_divideby"]) ncformat = config["OPTIONS"]["ncformat"] vicasc2routenc(filelist, domainfile, columns, period, outfiletemplate, ncvarinfo, ncformat) return
def parse_commandline(): parser = argparse.ArgumentParser(description='Update VIC state file ' 'based on assimilation of observed ' 'streamflow') parser.add_argument('--date', required=True, metavar='<date>', help='Forecast date: YYYY-MM-DD') parser.add_argument('--stations', required=True, metavar='<stations>', help='Comma-delimited list of stations to use ' 'in the update') parser.add_argument('--config', required=True, metavar='<configuration file>', help='Configuration file') args = parser.parse_args() forecastdate = parse_date(args.date, '-') configparser = ConfigParser.ConfigParser() configparser.read(os.path.expanduser(args.config)) config = {} for section in configparser.sections(): for key, value in configparser.items(section): config[key] = value config['timeofconcentration'] = int(config['timeofconcentration']) config['latlondigits'] = int(config['latlondigits']) stations = dict((station, config['timeofconcentration']) for station in args.stations.split(',')) return (forecastdate, stations, config)
import os import glob import pandas as pd from utils.parse import parse_date if len(sys.argv) < 6: print 'Must provide inpath, outpath, filemask, startdate and enddate' sys.exit() inpath = sys.argv[1] outpath = sys.argv[2] filemask = sys.argv[3] startdate = sys.argv[4] enddate = sys.argv[5] start = parse_date(startdate, '/') end = parse_date(enddate, '/') dates = pd.date_range(start, end, freq='D') filelist = glob.glob('{}/{}*'.format(inpath, filemask)) # all the files are the same, so we need to determine the indices only once infile = filelist[0] data = np.loadtxt(infile, dtype={'names': ('prcp', 'tmax', 'tmin', 'wind'), 'formats': ('f4', 'f4', 'f4', 'f4')}) # determine the number of months nyears = end.year - start.year + 1 # initialize empty arrays idx = np.empty((data['prcp'].size, nyears), dtype='bool')*False
def __init__(self, configparser, configurationfile): """Initialize a particle filter based on a configurationfile""" self.pool = None # Parse the configuration file self.configfile = configurationfile self.config = collections.OrderedDict() # preserve order of entries for section in configparser.sections(): self.config[section] = collections.OrderedDict() for key, value in configparser.items(section): self.config[section][key] = value if not self.config['COMPUTING']['poolsize']: self.poolsize = multiprocessing.cpu_count() else: self.poolsize = configparser.getint('COMPUTING', 'poolsize') self.ID = self.config['FILTER']['runid'] logging.info('Initializing particle filter: {}'.format(self.ID)) ncvarinfo = {} varlist = filterPick(self.config['VICASC2NC'].keys(), re.compile('^ncvar.*_name$')) varlist = [x.split('_name')[0] for x in varlist] for var in varlist: ncvarinfo[var] = {} ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name'] ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var + '_longname'] ncvarinfo[var]['column'] = self.config['VICASC2NC'][var + '_column'] ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units'] ncvarinfo[var]['divideby'] = float( self.config['VICASC2NC'][var + '_divideby']) self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo # initialize simulation time self.start = parse_date(self.config['SIMULATION']['start']) self.now = self.start self.end = parse_date(self.config['SIMULATION']['end']) self.runlength = dt.timedelta( configparser.getint('SIMULATION', 'runlength')) self.backstep = dt.timedelta( configparser.getint('SIMULATION', 'backstep')) # number of timestep to evaluate self.neval = configparser.getint('FILTER', 'neval') # Initialize particles self.np = configparser.getint('FILTER', 'np') self.particles = [ VicParticle(i + 1, 1. / self.np) for i in range(self.np) ] for i in range(self.np): p = self.particles[i] p.vic['exe'] = self.config['MODELS']['vicexe'] p.route['exe'] = self.config['MODELS']['routeexe'] p.refvar = self.config['FILTER']['referencevar'] p.initstatefile = {} p.initstatefile['vic'] = self.config['VICCONFIG'][ 'INIT_STATE'].format(p.ID) if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower( ) == 'startup': p.initstatefile['route'] = \ self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\ format(p.ID) p.neval = self.neval self.normalize_weights() logging.info('Initialized {} particles'.format(self.np)) # set up run and archive paths for particles runpath = self.config['FILTER']['rundir'] archivepath = self.config['FILTER']['archivedir'] subdirs = ['log', 'history', 'settings'] self.paths = {} self.paths['run'] = {} self.paths['archive'] = {} for subdir in subdirs: self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir) self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir) logging.debug('Creating paths for filter {}:'.format(self.ID)) for topdir, dirdict in self.paths.iteritems(): logging.debug('\t{:8s}:'.format(topdir)) for key, val in dirdict.iteritems(): logging.debug('\t\t{:6s}: {}'.format(key, val)) make_sure_path_exists(val) subdirs = ['config', 'data', 'log', 'state', 'history'] for p in self.particles: p.paths = {} p.paths['run'] = {} p.paths['archive'] = {} runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'], p.ID) archivepath = '{}/pid_{:03d}'.format( self.config['FILTER']['archivedir'], p.ID) for subdir in subdirs: p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir) p.paths['archive'][subdir] = '{}/{}'.format( archivepath, subdir) logging.debug('Creating paths for particle {:03d}:'.format(p.ID)) for topdir, dirdict in p.paths.iteritems(): logging.debug('\t{:8s}:'.format(topdir)) for key, val in dirdict.iteritems(): logging.debug('\t\t{:6s}: {}'.format(key, val)) make_sure_path_exists(val) # set history file for filter filename = '{}.history'.format(self.ID) self.hist = open(os.path.join(self.paths['run']['history'], filename), 'w') # set history file for each particle for p in self.particles: filename = '{}.{:03d}.history'.format(self.ID, p.ID) p.hist = open(os.path.join(p.paths['run']['history'], filename), 'w') logging.info('Initialization complete')
def __init__(self, configparser, configurationfile): """Initialize a particle filter based on a configurationfile""" self.pool = None # Parse the configuration file self.configfile = configurationfile self.config = collections.OrderedDict() # preserve order of entries for section in configparser.sections(): self.config[section] = collections.OrderedDict() for key, value in configparser.items(section): self.config[section][key] = value if not self.config['COMPUTING']['poolsize']: self.poolsize = multiprocessing.cpu_count() else: self.poolsize = configparser.getint('COMPUTING','poolsize') self.ID = self.config['FILTER']['runid'] logging.info('Initializing particle filter: {}'.format(self.ID)) ncvarinfo = {} varlist = filterPick(self.config['VICASC2NC'].keys(), re.compile('^ncvar.*_name$')) varlist = [x.split('_name')[0] for x in varlist] for var in varlist: ncvarinfo[var] = {} ncvarinfo[var]['name'] = self.config['VICASC2NC'][var + '_name'] ncvarinfo[var]['longname'] = self.config['VICASC2NC'][var + '_longname'] ncvarinfo[var]['column'] = self.config['VICASC2NC'][var + '_column'] ncvarinfo[var]['units'] = self.config['VICASC2NC'][var + '_units'] ncvarinfo[var]['divideby'] = float(self.config['VICASC2NC'][var + '_divideby']) self.config['VICASC2NC']['ncvarinfo'] = ncvarinfo # initialize simulation time self.start = parse_date(self.config['SIMULATION']['start']) self.now = self.start self.end = parse_date(self.config['SIMULATION']['end']) self.runlength = dt.timedelta(configparser.getint('SIMULATION', 'runlength')) self.backstep = dt.timedelta(configparser.getint('SIMULATION', 'backstep')) # number of timestep to evaluate self.neval = configparser.getint('FILTER','neval') # Initialize particles self.np = configparser.getint('FILTER', 'np') self.particles = [VicParticle(i+1, 1./self.np) for i in range(self.np)] for i in range(self.np): p = self.particles[i] p.vic['exe'] = self.config['MODELS']['vicexe'] p.route['exe'] = self.config['MODELS']['routeexe'] p.refvar = self.config['FILTER']['referencevar'] p.initstatefile = {} p.initstatefile['vic'] = self.config['VICCONFIG']['INIT_STATE'].format(p.ID) if self.config['ROUTECONFIG']['SECTION|OPTIONS|RUN_TYPE'].lower() == 'startup': p.initstatefile['route'] = \ self.config['ROUTECONFIG']['SECTION|INITIAL_STATE|FILE_NAME'].\ format(p.ID) p.neval = self.neval self.normalize_weights() logging.info('Initialized {} particles'.format(self.np)) # set up run and archive paths for particles runpath = self.config['FILTER']['rundir'] archivepath = self.config['FILTER']['archivedir'] subdirs = ['log', 'history', 'settings'] self.paths = {} self.paths['run'] = {} self.paths['archive'] = {} for subdir in subdirs: self.paths['run'][subdir] = '{}/{}'.format(runpath, subdir) self.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir) logging.debug('Creating paths for filter {}:'.format(self.ID)) for topdir, dirdict in self.paths.iteritems(): logging.debug('\t{:8s}:'.format(topdir)) for key, val in dirdict.iteritems(): logging.debug('\t\t{:6s}: {}'.format(key, val)) make_sure_path_exists(val) subdirs = ['config', 'data', 'log', 'state', 'history'] for p in self.particles: p.paths = {} p.paths['run'] = {} p.paths['archive'] = {} runpath = '{}/pid_{:03d}'.format(self.config['FILTER']['rundir'], p.ID) archivepath = '{}/pid_{:03d}'.format(self.config['FILTER']['archivedir'], p.ID) for subdir in subdirs: p.paths['run'][subdir] = '{}/{}'.format(runpath, subdir) p.paths['archive'][subdir] = '{}/{}'.format(archivepath, subdir) logging.debug('Creating paths for particle {:03d}:'.format(p.ID)) for topdir, dirdict in p.paths.iteritems(): logging.debug('\t{:8s}:'.format(topdir)) for key, val in dirdict.iteritems(): logging.debug('\t\t{:6s}: {}'.format(key, val)) make_sure_path_exists(val) # set history file for filter filename = '{}.history'.format(self.ID) self.hist = open(os.path.join(self.paths['run']['history'], filename), 'w') # set history file for each particle for p in self.particles: filename = '{}.{:03d}.history'.format(self.ID, p.ID) p.hist = open(os.path.join(p.paths['run']['history'], filename), 'w') logging.info('Initialization complete')