def edit_config_file(self, config_file, uhs_files, station_file, domain):
with open(station_file, "r") as f:
data = f.readlines()
data[1] = uhs_files
with open(station_file, "w") as f:
f.writelines(data)
parser = configparser.ConfigParser()
parser.optionxform = str
unprocessed = config_file
config_dict = read_config(unprocessed)
for section in config_dict.keys():
parser[section] = {
k: str(config_dict[section][k])
for k in config_dict[section].keys()
}
try:
parser["UHS_FILES"]["ROUT_DIR"] = "/".join(
uhs_files.split("/")[:-1])
parser["UHS_FILES"]["STATION_FILE"] = station_file
parser["DOMAIN"]["FILE_NAME"] = domain
except KeyError as e:
raise ProcessError(
f"{type(e).__name__}: Invalid header or config key in config file"
)
processed = ".".join(unprocessed.split(".")[:-1]) + "_edited.cfg"
with open(processed, "w") as cfg:
parser.write(cfg)
return processed
def _handler(self, request, response):
loglevel, uhs_files, station_file, domain, config_file = collect_args_wrapper(
request, self.workdir)
log_handler(
self,
response,
"Starting Process",
logger,
log_level=loglevel,
process_step="start",
)
log_handler(
self,
response,
"Rebuilding configuration",
logger,
log_level=loglevel,
process_step="config_rebuild",
)
config_file = self.edit_config_file(config_file, uhs_files,
station_file, domain)
log_handler(
self,
response,
"Run Parameter Conversion",
logger,
log_level=loglevel,
process_step="process",
)
try:
convert(config_file)
except Exception as e:
raise ProcessError(f"{type(e).__name__}: {e}")
log_handler(
self,
response,
"Building final output",
logger,
log_level=loglevel,
process_step="build_output",
)
config = read_config(config_file)
response.outputs["output"].file = get_outfile(config, "params")
log_handler(
self,
response,
"Process Complete",
logger,
log_level=loglevel,
process_step="complete",
)
return response
def run_examples(config_file):
""" Run examples from config file """
# ---------------------------------------------------------------- #
# Read Configuration files
config_dict = read_config(config_file)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# run tests
num_tests = len(config_dict.keys())
for i, (test, test_dict) in enumerate(config_dict.iteritems()):
print("".center(100, '-'))
print("Starting Test #{0} of {1}: {2}".format(i + 1, num_tests,
test).center(100))
desc = textwrap.fill(", ".join(test_dict['description']), 100)
print("Description: {0}".format(desc))
print("".center(100, '-'))
if 'processors' in test_dict:
numofproc = test_dict['processors']
else:
numofproc = 1
pr = cProfile.Profile()
pr.enable()
if test_dict['function'] == 'convert':
convert.convert(test_dict['config_file'])
elif test_dict['function'] == 'convolution':
convolution.convolution(test_dict['config_file'])
elif test_dict['function'] == 'parameters':
parameters.parameters(test_dict['config_file'],
numofproc=numofproc)
else:
raise ValueError('Unknow function variable: '
'{0}'.format(test_dict['function']))
pr.disable()
s = StringIO.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print("".center(100, '-'))
print("Done With Test #{0} of {1}: {2}".format(i + 1, num_tests,
test).center(100))
print(".....Printing Profile Information.....".center(100))
print("".center(100, '-'))
print(s.getvalue())
print("".center(100, '-'))
return
def run_examples(config_file):
""" Run examples from config file """
# ---------------------------------------------------------------- #
# Read Configuration files
config_dict = read_config(config_file)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# run tests
num_tests = len(config_dict.keys())
for i, (test, test_dict) in enumerate(config_dict.iteritems()):
print("".center(100, '-'))
print("Starting Test #{0} of {1}: {2}".format(i+1, num_tests,
test).center(100))
desc = textwrap.fill(", ".join(test_dict['description']), 100)
print("Description: {0}".format(desc))
print("".center(100, '-'))
if 'processors' in test_dict:
numofproc = test_dict['processors']
else:
numofproc = 1
pr = cProfile.Profile()
pr.enable()
if test_dict['function'] == 'convert':
convert.convert(test_dict['config_file'])
elif test_dict['function'] == 'convolution':
convolution.convolution(test_dict['config_file'])
elif test_dict['function'] == 'parameters':
parameters.parameters(test_dict['config_file'],
numofproc=numofproc)
else:
raise ValueError('Unknow function variable: '
'{0}'.format(test_dict['function']))
pr.disable()
s = StringIO.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print("".center(100, '-'))
print("Done With Test #{0} of {1}: {2}".format(i+1, num_tests,
test).center(100))
print(".....Printing Profile Information.....".center(100))
print("".center(100, '-'))
print(s.getvalue())
print("".center(100, '-'))
return
def convolve_config_handler(
workdir,
case_id,
run_startdate,
stop_date,
domain,
param_file,
input_forcings,
convolve_config_file,
convolve_config_dict,
):
if convolve_config_file:
unprocessed = read_config(convolve_config_file)
elif convolve_config_dict:
unprocessed = eval(convolve_config_dict)
else:
unprocessed = convolve_config_template
processed = convolve_config_template
try:
processed["OPTIONS"]["CASEID"] = case_id
processed["OPTIONS"]["RUN_STARTDATE"] = run_startdate
processed["OPTIONS"]["STOP_DATE"] = stop_date
for section in unprocessed.keys():
for key in unprocessed[section].keys():
processed[section][key] = unprocessed[section][key]
if not processed["OPTIONS"]["CASE_DIR"]:
processed["OPTIONS"]["CASE_DIR"] = os.path.join(
workdir, processed["OPTIONS"]["CASEID"])
if not processed["OPTIONS"]["REST_DATE"]:
processed["OPTIONS"]["REST_DATE"] = processed["OPTIONS"][
"STOP_DATE"]
processed["DOMAIN"]["FILE_NAME"] = domain
processed["PARAM_FILE"]["FILE_NAME"] = param_file
processed["INPUT_FORCINGS"]["DATL_PATH"] = "/".join(
input_forcings.split("/")[:-1])
processed["INPUT_FORCINGS"]["DATL_FILE"] = input_forcings.split(
"/")[-1]
return rvic_config_validator(processed)
except KeyError:
raise ProcessError("Invalid config key provided in convolve file ")
def params_config_handler(
workdir,
case_id,
domain,
grid_id,
pour_points,
routing,
uh_box,
params_config_file,
params_config_dict,
):
if params_config_file:
unprocessed = read_config(params_config_file)
elif params_config_dict:
unprocessed = eval(params_config_dict)
else:
unprocessed = params_config_template
processed = params_config_template
try:
processed["OPTIONS"]["CASEID"] = case_id
processed["OPTIONS"]["GRIDID"] = grid_id
for section in unprocessed.keys():
for key in unprocessed[section].keys():
processed[section][key] = unprocessed[section][key]
if processed["OPTIONS"]["CASE_DIR"] == None:
processed["OPTIONS"]["CASE_DIR"] = os.path.join(
workdir, processed["OPTIONS"]["CASEID"])
if processed["OPTIONS"]["TEMP_DIR"] == None:
processed["OPTIONS"][
"TEMP_DIR"] = processed["OPTIONS"]["CASEID"] + "/temp"
processed["POUR_POINTS"]["FILE_NAME"] = pour_points
processed["UH_BOX"]["FILE_NAME"] = uh_box
processed["ROUTING"]["FILE_NAME"] = routing
processed["DOMAIN"]["FILE_NAME"] = domain
return rvic_config_validator(processed)
except KeyError:
raise ProcessError("Invalid config key provided in param file")
import os
import pytest
from pathlib import Path
from pkg_resources import resource_filename
from rvic.convolution import convolution
from common import run_test, check_close_logger_call
from rvic.core.config import read_config
config_file = str(Path(__file__).parents[1]) + "/data/configs/convolve.cfg"
config_dict = read_config(config_file)
@pytest.mark.online
@pytest.mark.parametrize(
"config",
[config_file, config_dict],
)
def test_convolution(config):
convolution(config)
check_close_logger_call()
def test_invalid_input():
invalid_config = config_dict
invalid_config["INPUT_FORCINGS"]["DATL_PATH"] = "./tests/data/samples/"
with pytest.raises(FileNotFoundError):
convolution(invalid_config)
check_close_logger_call()
def run_examples(config_file):
''' Run examples from config file '''
# ---------------------------------------------------------------- #
# Read Configuration files
config_dict = read_config(config_file)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# run tests
num_tests = len(list(config_dict.keys()))
test_outcomes = OrderedDict()
for i, (test, test_dict) in enumerate(iteritems(config_dict)):
print(''.center(100, '-'))
print('Starting Test #{0} of {1}: {2}'.format(i + 1, num_tests,
test).center(100))
desc = textwrap.fill(', '.join(test_dict['description']), 100)
print('Description: {0}'.format(desc))
print(''.center(100, '-'))
if 'processors' in test_dict:
numofproc = test_dict['processors']
else:
numofproc = 1
pr = cProfile.Profile()
pr.enable()
if test_dict['function'] == 'convert':
try:
convert.convert(test_dict['config_file'])
test_outcomes[test] = 'Passed'
except Exception as e:
print('Error in conversion example: {0}'.format(test))
test_outcomes[test] = 'Failed: {0}'.format(e)
elif test_dict['function'] == 'convolution':
try:
convolution.convolution(test_dict['config_file'])
test_outcomes[test] = 'Passed'
except Exception as e:
print('Error in convolution example: {0}'.format(test))
test_outcomes[test] = 'Failed: {0}'.format(e)
elif test_dict['function'] == 'parameters':
try:
parameters.parameters(test_dict['config_file'],
numofproc=numofproc)
test_outcomes[test] = 'Passed'
except Exception as e:
print('Error in parameters example: {0}'.format(test))
test_outcomes[test] = 'Failed: {0}'.format(e)
else:
raise ValueError('Unknown function variable: '
'{0}'.format(test_dict['function']))
pr.disable()
if py3:
s = io.StringIO()
else:
s = io.BytesIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(''.center(100, '-'))
print('Done With Test #{0} of {1}: {2}'.format(i + 1, num_tests,
test).center(100))
print('.....Printing Profile Information.....'.center(100))
print(''.center(100, '-'))
print(s.getvalue())
print(''.center(100, '-'))
print('Done with examples...printing summary')
for test, outcome in iteritems(test_outcomes):
print('\t{0}: {1}'.format(test, outcome))
return 0
def run_examples(config_file):
''' Run examples from config file '''
# ---------------------------------------------------------------- #
# Read Configuration files
config_dict = read_config(config_file)
# ---------------------------------------------------------------- #
# ---------------------------------------------------------------- #
# run tests
num_tests = len(list(config_dict.keys()))
test_outcomes = OrderedDict()
for i, (test, test_dict) in enumerate(iteritems(config_dict)):
print(''.center(100, '-'))
print('Starting Test #{0} of {1}: {2}'.format(i + 1, num_tests,
test).center(100))
desc = textwrap.fill(', '.join(test_dict['description']), 100)
print('Description: {0}'.format(desc))
print(''.center(100, '-'))
if 'processors' in test_dict:
numofproc = test_dict['processors']
else:
numofproc = 1
pr = cProfile.Profile()
pr.enable()
if test_dict['function'] == 'convert':
try:
convert.convert(test_dict['config_file'])
test_outcomes[test] = 'Passed'
except Exception as e:
print('Error in conversion example: {0}'.format(test))
test_outcomes[test] = 'Failed: {0}'.format(e)
elif test_dict['function'] == 'convolution':
try:
convolution.convolution(test_dict['config_file'])
test_outcomes[test] = 'Passed'
except Exception as e:
print('Error in convolution example: {0}'.format(test))
test_outcomes[test] = 'Failed: {0}'.format(e)
elif test_dict['function'] == 'parameters':
try:
parameters.parameters(test_dict['config_file'],
numofproc=numofproc)
test_outcomes[test] = 'Passed'
except Exception as e:
print('Error in parameters example: {0}'.format(test))
test_outcomes[test] = 'Failed: {0}'.format(e)
else:
raise ValueError('Unknown function variable: '
'{0}'.format(test_dict['function']))
pr.disable()
if py3:
s = io.StringIO()
else:
s = io.BytesIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(''.center(100, '-'))
print('Done With Test #{0} of {1}: {2}'.format(i + 1, num_tests,
test).center(100))
print('.....Printing Profile Information.....'.center(100))
print(''.center(100, '-'))
print(s.getvalue())
print(''.center(100, '-'))
print('Done with examples...printing summary')
for test, outcome in iteritems(test_outcomes):
print('\t{0}: {1}'.format(test, outcome))
return 0