def test_relative_parameter_set(self): """Datastack: test relative parameter set.""" from natcap.invest import __version__ from natcap.invest import datastack params = { 'a': 1, 'b': 'hello there', 'c': 'plain bytestring', 'nested': { 'level1': 123, }, 'foo': os.path.join(self.workspace, 'foo.txt'), 'bar': os.path.join(self.workspace, 'foo.txt'), 'file_list': [ os.path.join(self.workspace, 'file1.txt'), os.path.join(self.workspace, 'file2.txt'), ], 'data_dir': os.path.join(self.workspace, 'data_dir'), 'temp_workspace': self.workspace } modelname = 'natcap.invest.foo' paramset_filename = os.path.join(self.workspace, 'paramset.json') # make the sample data so filepaths are interpreted correctly for file_base in ('foo', 'bar', 'file1', 'file2'): test_filepath = os.path.join(self.workspace, file_base + '.txt') open(test_filepath, 'w').write('hello!') os.makedirs(params['data_dir']) # Write the parameter set datastack.build_parameter_set(params, modelname, paramset_filename, relative=True) # Check that the written parameter set file contains relative paths raw_args = json.load(open(paramset_filename))['args'] self.assertEqual(raw_args['foo'], 'foo.txt') self.assertEqual(raw_args['bar'], 'foo.txt') self.assertEqual(raw_args['file_list'], ['file1.txt', 'file2.txt']) self.assertEqual(raw_args['data_dir'], 'data_dir') self.assertEqual(raw_args['temp_workspace'], '.') # Read back the parameter set and verify the returned paths are # absolute args, callable_name, invest_version = datastack.extract_parameter_set( paramset_filename) self.assertEqual(args, params) self.assertEqual(invest_version, __version__) self.assertEqual(callable_name, modelname)
def test_datastack_parameter_set(self): """Datastack: test datastack parameter set.""" from natcap.invest import __version__ from natcap.invest import datastack params = { 'a': 1, 'b': 'hello there', 'c': 'plain bytestring', 'd': 'true', 'nested': { 'level1': 123, }, 'foo': os.path.join(self.workspace, 'foo.txt'), 'bar': os.path.join(self.workspace, 'foo.txt'), 'file_list': [ os.path.join(self.workspace, 'file1.txt'), os.path.join(self.workspace, 'file2.txt'), ], 'data_dir': os.path.join(self.workspace, 'data_dir'), 'raster': os.path.join(DATA_DIR, 'dem'), 'vector': os.path.join(DATA_DIR, 'watersheds.shp'), 'table': os.path.join(DATA_DIR, 'carbon', 'carbon_pools_samp.csv'), } modelname = 'natcap.invest.foo' paramset_filename = os.path.join(self.workspace, 'paramset.json') # Write the parameter set datastack.build_parameter_set(params, modelname, paramset_filename) # Read back the parameter set args, callable_name, invest_version = datastack.extract_parameter_set( paramset_filename) # parameter set calculations normalizes all paths. # These are relative paths and must be patched. normalized_params = params.copy() normalized_params['d'] = True # should be read in as a bool for key in ('raster', 'vector', 'table'): normalized_params[key] = os.path.normpath(normalized_params[key]) self.assertEqual(args, normalized_params) self.assertEqual(invest_version, __version__) self.assertEqual(callable_name, modelname)
def test_relative_parameter_set_windows(self): """Datastack: test relative parameter set paths saved linux style.""" from natcap.invest import __version__ from natcap.invest import datastack params = { 'foo': os.path.join(self.workspace, 'foo.txt'), 'bar': os.path.join(self.workspace, 'inter_dir', 'bar.txt'), 'doh': os.path.join(self.workspace, 'inter_dir', 'inter_inter_dir', 'doh.txt'), 'data_dir': os.path.join(self.workspace, 'data_dir'), } os.makedirs( os.path.join(self.workspace, 'inter_dir', 'inter_inter_dir')) modelname = 'natcap.invest.foo' paramset_filename = os.path.join(self.workspace, 'paramset.json') # make the sample data so filepaths are interpreted correctly for base_name in ('foo', 'bar', 'doh'): open(params[base_name], 'w').write('hello!') os.makedirs(params['data_dir']) # Write the parameter set datastack.build_parameter_set(params, modelname, paramset_filename, relative=True) # Check that the written parameter set file contains relative paths raw_args = json.load(open(paramset_filename))['args'] self.assertEqual(raw_args['foo'], 'foo.txt') # Expecting linux style path separators for Windows self.assertEqual(raw_args['bar'], 'inter_dir/bar.txt') self.assertEqual(raw_args['doh'], 'inter_dir/inter_inter_dir/doh.txt') self.assertEqual(raw_args['data_dir'], 'data_dir') # Read back the parameter set and verify the returned paths are # absolute args, callable_name, invest_version = datastack.extract_parameter_set( paramset_filename) self.assertEqual(args, params) self.assertEqual(invest_version, __version__) self.assertEqual(callable_name, modelname)
def test_get_datatack_info_parameter_set(self): import natcap.invest from natcap.invest import datastack params = { 'a': 1, 'b': u'hello there', 'c': 'plain bytestring', 'd': '', } json_path = os.path.join(self.workspace, 'archive.invs.json') datastack.build_parameter_set(params, 'sample_model', json_path) stack_type, stack_info = datastack.get_datastack_info(json_path) self.assertEqual(stack_type, 'json') self.assertEqual(stack_info, datastack.ParameterSet( params, 'sample_model', natcap.invest.__version__))
def test_mixed_path_separators_in_paramset_windows(self): """Datastacks: parameter sets must handle windows and linux paths.""" from natcap.invest import datastack args = { 'windows_path': os.path.join(self.workspace, 'dir1\\filepath1.txt'), 'linux_path': os.path.join(self.workspace, 'dir2/filepath2.txt'), } for filepath in args.values(): normalized_path = os.path.normpath(filepath.replace('\\', os.sep)) try: os.makedirs(os.path.dirname(normalized_path)) except OSError: pass with open(normalized_path, 'w') as open_file: open_file.write('the contents of this file do not matter.') paramset_path = os.path.join(self.workspace, 'paramset.invest.json') # Windows paths should be saved with linux-style separators datastack.build_parameter_set(args, 'sample_model', paramset_path, relative=True) with open(paramset_path) as saved_parameters: args = json.loads(saved_parameters.read())['args'] # Expecting window_path to have linux style line seps expected_args = { 'windows_path': 'dir1/filepath1.txt', 'linux_path': 'dir2/filepath2.txt', } self.assertEqual(expected_args, args) expected_args = { 'windows_path': os.path.join(self.workspace, 'dir1', 'filepath1.txt'), 'linux_path': os.path.join(self.workspace, 'dir2', 'filepath2.txt'), } extracted_paramset = datastack.extract_parameter_set(paramset_path) self.assertEqual(extracted_paramset.args, expected_args)
def test_mixed_path_separators_in_paramset_mac(self): """Datastacks: parameter sets must handle mac and linux paths.""" from natcap.invest import datastack args = { 'mac_path': os.path.join(self.workspace, 'dir1/filepath1.txt'), 'linux_path': os.path.join(self.workspace, 'dir2/filepath2.txt'), } for filepath in args.values(): try: os.makedirs(os.path.dirname(filepath)) except OSError: pass with open(filepath, 'w') as open_file: open_file.write('the contents of this file do not matter.') paramset_path = os.path.join(self.workspace, 'paramset.invest.json') datastack.build_parameter_set(args, 'sample_model', paramset_path, relative=True) with open(paramset_path) as saved_parameters: args = json.loads(saved_parameters.read())['args'] expected_args = { 'mac_path': 'dir1/filepath1.txt', 'linux_path': 'dir2/filepath2.txt', } self.assertEqual(expected_args, args) expected_args = { 'mac_path': os.path.join(self.workspace, 'dir1', 'filepath1.txt'), 'linux_path': os.path.join(self.workspace, 'dir2', 'filepath2.txt'), } extracted_paramset = datastack.extract_parameter_set(paramset_path) self.assertEqual(extracted_paramset.args, expected_args)
def test_get_datastack_info_parameter_set(self): """Datastack: test get datastack info parameter set.""" import natcap.invest from natcap.invest import datastack params = { 'a': 1, 'b': 'hello there', 'c': 'plain bytestring', 'd': '', } test_module_name = 'test_datastack_modules.simple_parameters' json_path = os.path.join(self.workspace, 'archive.invs.json') datastack.build_parameter_set(params, test_module_name, json_path) stack_type, stack_info = datastack.get_datastack_info(json_path) self.assertEqual(stack_type, 'json') self.assertEqual( stack_info, datastack.ParameterSet(params, test_module_name, natcap.invest.__version__))
def write_parameter_set_file(): """Writes InVEST model args keys and values to a datastack JSON file. Body (JSON string): parameterSetPath: string moduleName: string(e.g. natcap.invest.carbon) args: JSON string of InVEST model args keys and values relativePaths: boolean Returns: A string. """ payload = request.get_json() filepath = payload['parameterSetPath'] modulename = payload['moduleName'] args = json.loads(payload['args']) relative_paths = payload['relativePaths'] datastack.build_parameter_set(args, modulename, filepath, relative=relative_paths) return 'parameter set saved'