def test_relative_parameter_set(self):
"""Datastack: test relative parameter set."""
from natcap.invest import __version__
from natcap.invest import datastack
params = {
'a':
1,
'b':
'hello there',
'c':
'plain bytestring',
'nested': {
'level1': 123,
},
'foo':
os.path.join(self.workspace, 'foo.txt'),
'bar':
os.path.join(self.workspace, 'foo.txt'),
'file_list': [
os.path.join(self.workspace, 'file1.txt'),
os.path.join(self.workspace, 'file2.txt'),
],
'data_dir':
os.path.join(self.workspace, 'data_dir'),
'temp_workspace':
self.workspace
}
modelname = 'natcap.invest.foo'
paramset_filename = os.path.join(self.workspace, 'paramset.json')
# make the sample data so filepaths are interpreted correctly
for file_base in ('foo', 'bar', 'file1', 'file2'):
test_filepath = os.path.join(self.workspace, file_base + '.txt')
open(test_filepath, 'w').write('hello!')
os.makedirs(params['data_dir'])
# Write the parameter set
datastack.build_parameter_set(params,
modelname,
paramset_filename,
relative=True)
# Check that the written parameter set file contains relative paths
raw_args = json.load(open(paramset_filename))['args']
self.assertEqual(raw_args['foo'], 'foo.txt')
self.assertEqual(raw_args['bar'], 'foo.txt')
self.assertEqual(raw_args['file_list'], ['file1.txt', 'file2.txt'])
self.assertEqual(raw_args['data_dir'], 'data_dir')
self.assertEqual(raw_args['temp_workspace'], '.')
# Read back the parameter set and verify the returned paths are
# absolute
args, callable_name, invest_version = datastack.extract_parameter_set(
paramset_filename)
self.assertEqual(args, params)
self.assertEqual(invest_version, __version__)
self.assertEqual(callable_name, modelname)
def test_datastack_parameter_set(self):
"""Datastack: test datastack parameter set."""
from natcap.invest import __version__
from natcap.invest import datastack
params = {
'a':
1,
'b':
'hello there',
'c':
'plain bytestring',
'd':
'true',
'nested': {
'level1': 123,
},
'foo':
os.path.join(self.workspace, 'foo.txt'),
'bar':
os.path.join(self.workspace, 'foo.txt'),
'file_list': [
os.path.join(self.workspace, 'file1.txt'),
os.path.join(self.workspace, 'file2.txt'),
],
'data_dir':
os.path.join(self.workspace, 'data_dir'),
'raster':
os.path.join(DATA_DIR, 'dem'),
'vector':
os.path.join(DATA_DIR, 'watersheds.shp'),
'table':
os.path.join(DATA_DIR, 'carbon', 'carbon_pools_samp.csv'),
}
modelname = 'natcap.invest.foo'
paramset_filename = os.path.join(self.workspace, 'paramset.json')
# Write the parameter set
datastack.build_parameter_set(params, modelname, paramset_filename)
# Read back the parameter set
args, callable_name, invest_version = datastack.extract_parameter_set(
paramset_filename)
# parameter set calculations normalizes all paths.
# These are relative paths and must be patched.
normalized_params = params.copy()
normalized_params['d'] = True # should be read in as a bool
for key in ('raster', 'vector', 'table'):
normalized_params[key] = os.path.normpath(normalized_params[key])
self.assertEqual(args, normalized_params)
self.assertEqual(invest_version, __version__)
self.assertEqual(callable_name, modelname)
def test_relative_parameter_set_windows(self):
"""Datastack: test relative parameter set paths saved linux style."""
from natcap.invest import __version__
from natcap.invest import datastack
params = {
'foo':
os.path.join(self.workspace, 'foo.txt'),
'bar':
os.path.join(self.workspace, 'inter_dir', 'bar.txt'),
'doh':
os.path.join(self.workspace, 'inter_dir', 'inter_inter_dir',
'doh.txt'),
'data_dir':
os.path.join(self.workspace, 'data_dir'),
}
os.makedirs(
os.path.join(self.workspace, 'inter_dir', 'inter_inter_dir'))
modelname = 'natcap.invest.foo'
paramset_filename = os.path.join(self.workspace, 'paramset.json')
# make the sample data so filepaths are interpreted correctly
for base_name in ('foo', 'bar', 'doh'):
open(params[base_name], 'w').write('hello!')
os.makedirs(params['data_dir'])
# Write the parameter set
datastack.build_parameter_set(params,
modelname,
paramset_filename,
relative=True)
# Check that the written parameter set file contains relative paths
raw_args = json.load(open(paramset_filename))['args']
self.assertEqual(raw_args['foo'], 'foo.txt')
# Expecting linux style path separators for Windows
self.assertEqual(raw_args['bar'], 'inter_dir/bar.txt')
self.assertEqual(raw_args['doh'], 'inter_dir/inter_inter_dir/doh.txt')
self.assertEqual(raw_args['data_dir'], 'data_dir')
# Read back the parameter set and verify the returned paths are
# absolute
args, callable_name, invest_version = datastack.extract_parameter_set(
paramset_filename)
self.assertEqual(args, params)
self.assertEqual(invest_version, __version__)
self.assertEqual(callable_name, modelname)
def test_get_datatack_info_parameter_set(self):
import natcap.invest
from natcap.invest import datastack
params = {
'a': 1,
'b': u'hello there',
'c': 'plain bytestring',
'd': '',
}
json_path = os.path.join(self.workspace, 'archive.invs.json')
datastack.build_parameter_set(params, 'sample_model', json_path)
stack_type, stack_info = datastack.get_datastack_info(json_path)
self.assertEqual(stack_type, 'json')
self.assertEqual(stack_info, datastack.ParameterSet(
params, 'sample_model', natcap.invest.__version__))
def test_mixed_path_separators_in_paramset_windows(self):
"""Datastacks: parameter sets must handle windows and linux paths."""
from natcap.invest import datastack
args = {
'windows_path': os.path.join(self.workspace,
'dir1\\filepath1.txt'),
'linux_path': os.path.join(self.workspace, 'dir2/filepath2.txt'),
}
for filepath in args.values():
normalized_path = os.path.normpath(filepath.replace('\\', os.sep))
try:
os.makedirs(os.path.dirname(normalized_path))
except OSError:
pass
with open(normalized_path, 'w') as open_file:
open_file.write('the contents of this file do not matter.')
paramset_path = os.path.join(self.workspace, 'paramset.invest.json')
# Windows paths should be saved with linux-style separators
datastack.build_parameter_set(args,
'sample_model',
paramset_path,
relative=True)
with open(paramset_path) as saved_parameters:
args = json.loads(saved_parameters.read())['args']
# Expecting window_path to have linux style line seps
expected_args = {
'windows_path': 'dir1/filepath1.txt',
'linux_path': 'dir2/filepath2.txt',
}
self.assertEqual(expected_args, args)
expected_args = {
'windows_path': os.path.join(self.workspace, 'dir1',
'filepath1.txt'),
'linux_path': os.path.join(self.workspace, 'dir2',
'filepath2.txt'),
}
extracted_paramset = datastack.extract_parameter_set(paramset_path)
self.assertEqual(extracted_paramset.args, expected_args)
def test_mixed_path_separators_in_paramset_mac(self):
"""Datastacks: parameter sets must handle mac and linux paths."""
from natcap.invest import datastack
args = {
'mac_path': os.path.join(self.workspace, 'dir1/filepath1.txt'),
'linux_path': os.path.join(self.workspace, 'dir2/filepath2.txt'),
}
for filepath in args.values():
try:
os.makedirs(os.path.dirname(filepath))
except OSError:
pass
with open(filepath, 'w') as open_file:
open_file.write('the contents of this file do not matter.')
paramset_path = os.path.join(self.workspace, 'paramset.invest.json')
datastack.build_parameter_set(args,
'sample_model',
paramset_path,
relative=True)
with open(paramset_path) as saved_parameters:
args = json.loads(saved_parameters.read())['args']
expected_args = {
'mac_path': 'dir1/filepath1.txt',
'linux_path': 'dir2/filepath2.txt',
}
self.assertEqual(expected_args, args)
expected_args = {
'mac_path': os.path.join(self.workspace, 'dir1', 'filepath1.txt'),
'linux_path': os.path.join(self.workspace, 'dir2',
'filepath2.txt'),
}
extracted_paramset = datastack.extract_parameter_set(paramset_path)
self.assertEqual(extracted_paramset.args, expected_args)
def test_get_datastack_info_parameter_set(self):
"""Datastack: test get datastack info parameter set."""
import natcap.invest
from natcap.invest import datastack
params = {
'a': 1,
'b': 'hello there',
'c': 'plain bytestring',
'd': '',
}
test_module_name = 'test_datastack_modules.simple_parameters'
json_path = os.path.join(self.workspace, 'archive.invs.json')
datastack.build_parameter_set(params, test_module_name, json_path)
stack_type, stack_info = datastack.get_datastack_info(json_path)
self.assertEqual(stack_type, 'json')
self.assertEqual(
stack_info,
datastack.ParameterSet(params, test_module_name,
natcap.invest.__version__))
def write_parameter_set_file():
"""Writes InVEST model args keys and values to a datastack JSON file.
Body (JSON string):
parameterSetPath: string
moduleName: string(e.g. natcap.invest.carbon)
args: JSON string of InVEST model args keys and values
relativePaths: boolean
Returns:
A string.
"""
payload = request.get_json()
filepath = payload['parameterSetPath']
modulename = payload['moduleName']
args = json.loads(payload['args'])
relative_paths = payload['relativePaths']
datastack.build_parameter_set(args,
modulename,
filepath,
relative=relative_paths)
return 'parameter set saved'
