def test_prepare_workspace(self):
"""utils: test that prepare_workspace does what is expected."""
from natcap.invest import utils
workspace = os.path.join(self.workspace, 'foo')
try:
with utils.prepare_workspace(workspace, 'some_model'):
warnings.warn('deprecated', UserWarning)
gdal.Open('file should not exist')
except Warning as warning_raised:
self.fail('Warning was not captured: %s' % warning_raised)
self.assertTrue(os.path.exists(workspace))
logfile_glob = glob.glob(os.path.join(workspace, '*.txt'))
self.assertEqual(len(logfile_glob), 1)
self.assertTrue(
os.path.basename(logfile_glob[0]).startswith('InVEST-some_model'))
with open(logfile_glob[0]) as logfile:
logfile_text = logfile.read()
# all the following strings should be in the logfile.
expected_string = 'file should not exist: No such file or directory'
self.assertTrue(expected_string
in logfile_text) # gdal error captured
self.assertEqual(len(re.findall('WARNING', logfile_text)), 1)
self.assertTrue('Elapsed time:' in logfile_text)
def main(user_args=None):
"""CLI entry point for launching InVEST runs.
This command-line interface supports two methods of launching InVEST models
from the command-line:
* through its GUI
* in headless mode, without its GUI.
Running in headless mode allows us to bypass all GUI functionality,
so models may be run in this way without having GUI packages
installed.
"""
parser = argparse.ArgumentParser(description=(
'Integrated Valuation of Ecosystem Services and Tradeoffs. '
'InVEST (Integrated Valuation of Ecosystem Services and '
'Tradeoffs) is a family of tools for quantifying the values of '
'natural capital in clear, credible, and practical ways. In '
'promising a return (of societal benefits) on investments in '
'nature, the scientific community needs to deliver knowledge and '
'tools to quantify and forecast this return. InVEST enables '
'decision-makers to quantify the importance of natural capital, '
'to assess the tradeoffs associated with alternative choices, '
'and to integrate conservation and human development. \n\n'
'Older versions of InVEST ran as script tools in the ArcGIS '
'ArcToolBox environment, but have almost all been ported over to '
'a purely open-source python environment.'),
prog='invest')
parser.add_argument('--version', action='version', version=__version__)
verbosity_group = parser.add_mutually_exclusive_group()
verbosity_group.add_argument(
'-v',
'--verbose',
dest='verbosity',
default=0,
action='count',
help=('Increase verbosity. Affects how much logging is printed to '
'the console and (if running in headless mode) how much is '
'written to the logfile.'))
verbosity_group.add_argument('--debug',
dest='log_level',
default=logging.CRITICAL,
action='store_const',
const=logging.DEBUG,
help='Enable debug logging. Alias for -vvvvv')
subparsers = parser.add_subparsers(dest='subcommand')
listmodels_subparser = subparsers.add_parser(
'list', help='List the available InVEST models')
listmodels_subparser.add_argument('--json',
action='store_true',
help='Write output as a JSON object')
subparsers.add_parser('launch', help='Start the InVEST launcher window')
run_subparser = subparsers.add_parser('run', help='Run an InVEST model')
run_subparser.add_argument('-l',
'--headless',
action='store_true',
help=('Run an InVEST model without its GUI. '
'Requires a datastack and a workspace.'))
run_subparser.add_argument(
'-d',
'--datastack',
default=None,
nargs='?',
help=('Run the specified model with this JSON datastack. '
'Required if using --headless'))
run_subparser.add_argument(
'-w',
'--workspace',
default=None,
nargs='?',
help=('The workspace in which outputs will be saved. '
'Required if using --headless'))
run_subparser.add_argument(
'model',
action=SelectModelAction, # Assert valid model name
help=('The model to run. Use "invest list" to list the available '
'models.'))
quickrun_subparser = subparsers.add_parser(
'quickrun',
help=('Run through a model with a specific datastack, exiting '
'immediately upon completion. This subcommand is only intended '
'to be used by automated testing scripts.'))
quickrun_subparser.add_argument(
'model',
action=SelectModelAction, # Assert valid model name
help=('The model to run. Use "invest list" to list the available '
'models.'))
quickrun_subparser.add_argument(
'datastack', help=('Run the model with this JSON datastack.'))
quickrun_subparser.add_argument(
'-w',
'--workspace',
default=None,
nargs='?',
help=('The workspace in which outputs will be saved.'))
validate_subparser = subparsers.add_parser(
'validate', help=('Validate the parameters of a datastack'))
validate_subparser.add_argument('--json',
action='store_true',
help='Write output as a JSON object')
validate_subparser.add_argument(
'datastack', help=('Run the model with this JSON datastack.'))
getspec_subparser = subparsers.add_parser(
'getspec', help=('Get the specification of a model.'))
getspec_subparser.add_argument('--json',
action='store_true',
help='Write output as a JSON object')
getspec_subparser.add_argument(
'model',
action=SelectModelAction, # Assert valid model name
help=('The model for which the spec should be fetched. Use "invest '
'list" to list the available models.'))
args = parser.parse_args(user_args)
root_logger = logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
fmt='%(asctime)s %(name)-18s %(levelname)-8s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S ')
handler.setFormatter(formatter)
# Set the log level based on what the user provides in the available
# arguments. Verbosity: the more v's the lower the logging threshold.
# If --debug is used, the logging threshold is 10.
# If the user goes lower than logging.DEBUG, default to logging.DEBUG.
log_level = min(args.log_level, logging.CRITICAL - (args.verbosity * 10))
handler.setLevel(max(log_level, logging.DEBUG)) # don't go below DEBUG
root_logger.addHandler(handler)
LOGGER.info('Setting handler log level to %s', log_level)
# FYI: Root logger by default has a level of logging.WARNING.
# To capture ALL logging produced in this system at runtime, use this:
# logging.getLogger().setLevel(logging.DEBUG)
# Also FYI: using logging.DEBUG means that the logger will defer to
# the setting of the parent logger.
logging.getLogger('natcap').setLevel(logging.DEBUG)
if args.subcommand == 'list':
if args.json:
message = build_model_list_json()
else:
message = build_model_list_table()
sys.stdout.write(message)
parser.exit()
if args.subcommand == 'launch':
from natcap.invest.ui import launcher
parser.exit(launcher.main())
if args.subcommand == 'validate':
try:
parsed_datastack = datastack.extract_parameter_set(args.datastack)
except Exception as error:
parser.exit(
1, "Error when parsing JSON datastack:\n " + str(error))
model_module = importlib.import_module(
name=parsed_datastack.model_name)
try:
validation_result = getattr(model_module,
'validate')(parsed_datastack.args)
except KeyError as missing_keys_error:
if args.json:
message = json.dumps({
'validation_results': {
str(list(missing_keys_error.args)): 'Key is missing'
}
})
else:
message = ('Datastack is missing keys:\n ' +
str(missing_keys_error.args))
# Missing keys have an exit code of 1 because that would indicate
# probably programmer error.
sys.stdout.write(message)
parser.exit(1)
except Exception as error:
parser.exit(
1, ('Datastack could not be validated:\n ' + str(error)))
# Even validation errors will have an exit code of 0
if args.json:
message = json.dumps({'validation_results': validation_result})
else:
message = pprint.pformat(validation_result)
sys.stdout.write(message)
parser.exit(0)
if args.subcommand == 'getspec':
target_model = _MODEL_UIS[args.model].pyname
model_module = importlib.import_module(name=target_model)
spec = model_module.ARGS_SPEC
if args.json:
message = json.dumps(spec)
else:
message = pprint.pformat(spec)
sys.stdout.write(message)
parser.exit(0)
if args.subcommand == 'run' and args.headless:
if not args.datastack:
parser.exit(1, 'Datastack required for headless execution.')
try:
parsed_datastack = datastack.extract_parameter_set(args.datastack)
except Exception as error:
parser.exit(
1, "Error when parsing JSON datastack:\n " + str(error))
if not args.workspace:
if ('workspace_dir' not in parsed_datastack.args
or parsed_datastack.args['workspace_dir'] in ['', None]):
parser.exit(1,
('Workspace must be defined at the command line '
'or in the datastack file'))
else:
parsed_datastack.args['workspace_dir'] = args.workspace
target_model = _MODEL_UIS[args.model].pyname
model_module = importlib.import_module(name=target_model)
LOGGER.info('Imported target %s from %s', model_module.__name__,
model_module)
with utils.prepare_workspace(parsed_datastack.args['workspace_dir'],
name=parsed_datastack.model_name,
logging_level=log_level):
LOGGER.log(
datastack.ARGS_LOG_LEVEL,
'Starting model with parameters: \n%s',
datastack.format_args_dict(parsed_datastack.args,
parsed_datastack.model_name))
# We're deliberately not validating here because the user
# can just call ``invest validate <datastack>`` to validate.
getattr(model_module, 'execute')(parsed_datastack.args)
# If we're running in a GUI (either through ``invest run`` or
# ``invest quickrun``), we'll need to load the Model's GUI class,
# populate parameters and then (if in a quickrun) exit when the model
# completes. Quickrun functionality is primarily useful for automated
# testing of the model interfaces.
if (args.subcommand == 'run' and not args.headless
or args.subcommand == 'quickrun'):
# Creating this warning for future us to alert us to potential issues
# if/when we forget to define QT_MAC_WANTS_LAYER at runtime.
if (platform.system() == "Darwin"
and "QT_MAC_WANTS_LAYER" not in os.environ):
warnings.warn(
"Mac OS X Big Sur may require the 'QT_MAC_WANTS_LAYER' "
"environment variable to be defined in order to run. If "
"the application hangs on startup, set 'QT_MAC_WANTS_LAYER=1' "
"in the shell running this CLI.", RuntimeWarning)
from natcap.invest.ui import inputs
gui_class = _MODEL_UIS[args.model].gui
module_name, classname = gui_class.split('.')
module = importlib.import_module(name='.ui.%s' % module_name,
package='natcap.invest')
# Instantiate the form
model_form = getattr(module, classname)()
# load the datastack if one was provided
try:
if args.datastack:
model_form.load_datastack(args.datastack)
except Exception as error:
# If we encounter an exception while loading the datastack, log the
# exception (so it can be seen if we're running with appropriate
# verbosity) and exit the argparse application with exit code 1 and
# a helpful error message.
LOGGER.exception('Could not load datastack')
parser.exit(DEFAULT_EXIT_CODE,
'Could not load datastack: %s\n' % str(error))
if args.workspace:
model_form.workspace.set_value(args.workspace)
# Run the UI's event loop
quickrun = False
if args.subcommand == 'quickrun':
quickrun = True
model_form.run(quickrun=quickrun)
app_exitcode = inputs.QT_APP.exec_()
# Handle a graceful exit
if model_form.form.run_dialog.messageArea.error:
parser.exit(DEFAULT_EXIT_CODE,
'Model %s: run failed\n' % args.model)
if app_exitcode != 0:
parser.exit(app_exitcode,
'App terminated with exit code %s\n' % app_exitcode)
def main():
"""CLI entry point for launching InVEST runs.
This command-line interface supports two methods of launching InVEST models
from the command-line:
* through its GUI
* in headless mode, without its GUI.
Running in headless mode allows us to bypass all GUI functionality,
so models may be run in this way wthout having GUI packages
installed.
"""
parser = argparse.ArgumentParser(description=(
'Integrated Valuation of Ecosystem Services and Tradeoffs. '
'InVEST (Integrated Valuation of Ecosystem Services and Tradeoffs) is '
'a family of tools for quantifying the values of natural capital in '
'clear, credible, and practical ways. In promising a return (of '
'societal benefits) on investments in nature, the scientific community '
'needs to deliver knowledge and tools to quantify and forecast this '
'return. InVEST enables decision-makers to quantify the importance of '
'natural capital, to assess the tradeoffs associated with alternative '
'choices, and to integrate conservation and human development. \n\n'
'Older versions of InVEST ran as script tools in the ArcGIS ArcToolBox '
'environment, but have almost all been ported over to a purely '
'open-source python environment.'),
prog='invest')
list_group = parser.add_mutually_exclusive_group()
verbosity_group = parser.add_mutually_exclusive_group()
import natcap.invest
parser.add_argument('--version',
action='version',
version=natcap.invest.__version__)
verbosity_group.add_argument(
'-v',
'--verbose',
dest='verbosity',
default=0,
action='count',
help=('Increase verbosity. Affects how much is '
'printed to the console and (if running '
'in headless mode) how much is written '
'to the logfile.'))
verbosity_group.add_argument('--debug',
dest='log_level',
default=logging.CRITICAL,
action='store_const',
const=logging.DEBUG,
help='Enable debug logging. Alias for -vvvvv')
list_group.add_argument('--list',
action=ListModelsAction,
nargs=0,
const=True,
help='List available models')
parser.add_argument('-l',
'--headless',
action='store_true',
dest='headless',
help=('Attempt to run InVEST without its GUI.'))
parser.add_argument('-d',
'--datastack',
default=None,
nargs='?',
help='Run the specified model with this datastack')
parser.add_argument('-w',
'--workspace',
default=None,
nargs='?',
help='The workspace in which outputs will be saved')
gui_options_group = parser.add_argument_group(
'gui options', 'These options are ignored if running in headless mode')
gui_options_group.add_argument('-q',
'--quickrun',
action='store_true',
help=('Run the target model without '
'validating and quit with a nonzero '
'exit status if an exception is '
'encountered'))
cli_options_group = parser.add_argument_group('headless options')
cli_options_group.add_argument('-y',
'--overwrite',
action='store_true',
default=False,
help=('Overwrite the workspace without '
'prompting for confirmation'))
cli_options_group.add_argument('-n',
'--no-validate',
action='store_true',
dest='validate',
default=True,
help=('Do not validate inputs before '
'running the model.'))
list_group.add_argument('model',
action=SelectModelAction,
nargs='?',
help=('The model/tool to run. Use --list to show '
'available models/tools. Identifiable model '
'prefixes may also be used. Alternatively,'
'specify "launcher" to reveal a model '
'launcher window.'))
args = parser.parse_args()
root_logger = logging.getLogger()
handler = logging.StreamHandler(sys.stdout)
formatter = logging.Formatter(
fmt='%(asctime)s %(name)-18s %(levelname)-8s %(message)s',
datefmt='%m/%d/%Y %H:%M:%S ')
handler.setFormatter(formatter)
# Set the log level based on what the user provides in the available
# arguments. Verbosity: the more v's the lower the logging threshold.
# If --debug is used, the logging threshold is 10.
# If the user goes lower than logging.DEBUG, default to logging.DEBUG.
log_level = min(args.log_level, logging.CRITICAL - (args.verbosity * 10))
handler.setLevel(max(log_level,
logging.DEBUG)) # don't go lower than DEBUG
root_logger.addHandler(handler)
LOGGER.info('Setting handler log level to %s', log_level)
# FYI: Root logger by default has a level of logging.WARNING.
# To capture ALL logging produced in this system at runtime, use this:
# logging.getLogger().setLevel(logging.DEBUG)
# Also FYI: using logging.DEBUG means that the logger will defer to
# the setting of the parent logger.
logging.getLogger('natcap').setLevel(logging.DEBUG)
# Now that we've set up logging based on args, we can start logging.
LOGGER.debug(args)
try:
# Importing model UI files here will usually import qtpy before we can
# set the sip API in natcap.invest.ui.inputs.
# Set it here, before we can do the actual importing.
import sip
# 2 indicates SIP/Qt API version 2
sip.setapi('QString', 2)
from natcap.invest.ui import inputs
except ImportError as error:
# Can't import UI, exit with nonzero exit code
LOGGER.exception('Unable to import the UI')
parser.error(('Unable to import the UI (failed with "%s")\n'
'Is the UI installed?\n'
' pip install natcap.invest[ui]') % error)
if args.model == 'launcher':
from natcap.invest.ui import launcher
launcher.main()
elif args.headless:
from natcap.invest import datastack
target_mod = _MODEL_UIS[args.model].pyname
model_module = importlib.import_module(name=target_mod)
LOGGER.info('imported target %s from %s', model_module.__name__,
model_module)
paramset = datastack.extract_parameter_set(args.datastack)
# prefer CLI option for workspace dir, but use paramset workspace if
# the CLI options do not define a workspace.
if args.workspace:
workspace = os.path.abspath(args.workspace)
paramset.args['workspace_dir'] = workspace
else:
if 'workspace_dir' in paramset.args:
workspace = paramset.args['workspace_dir']
else:
parser.exit(DEFAULT_EXIT_CODE,
('Workspace not defined. \n'
'Use --workspace to specify or add a '
'"workspace_dir" parameter to your datastack.'))
with utils.prepare_workspace(workspace,
name=paramset.model_name,
logging_level=log_level):
LOGGER.log(
datastack.ARGS_LOG_LEVEL,
datastack.format_args_dict(paramset.args, paramset.model_name))
if not args.validate:
LOGGER.info('Skipping validation by user request')
else:
model_warnings = []
try:
model_warnings = getattr(model_module,
'validate')(paramset.args)
except AttributeError:
LOGGER.warn(
'%s does not have a defined validation function.',
paramset.model_name)
finally:
if model_warnings:
LOGGER.warn('Warnings found: \n%s',
pprint.pformat(model_warnings))
if not args.workspace:
args.workspace = os.getcwd()
# If the workspace exists and we don't have up-front permission to
# overwrite the workspace, prompt for permission.
if (os.path.exists(args.workspace)
and len(os.listdir(args.workspace)) > 0
and not args.overwrite):
overwrite_denied = False
if not sys.stdout.isatty():
overwrite_denied = True
else:
user_response = raw_input(
'Workspace exists: %s\n Overwrite? (y/n) ' %
(os.path.abspath(args.workspace)))
while user_response not in ('y', 'n'):
user_response = raw_input(
"Response must be either 'y' or 'n': ")
if user_response == 'n':
overwrite_denied = True
if overwrite_denied:
# Exit the parser with an error message.
parser.exit(DEFAULT_EXIT_CODE,
('Use --workspace to define an '
'alternate workspace. Aborting.'))
else:
LOGGER.warning(
'Overwriting the workspace per user input %s',
os.path.abspath(args.workspace))
if 'workspace_dir' not in paramset.args:
paramset.args['workspace_dir'] = args.workspace
# execute the model's execute function with the loaded args
getattr(model_module, 'execute')(paramset.args)
else:
# import the GUI from the known class
gui_class = _MODEL_UIS[args.model].gui
module_name, classname = gui_class.split('.')
module = importlib.import_module(name='.ui.%s' % module_name,
package='natcap.invest')
# Instantiate the form
model_form = getattr(module, classname)()
# load the datastack if one was provided
try:
if args.datastack:
model_form.load_datastack(args.datastack)
except Exception as error:
# If we encounter an exception while loading the datastack, log the
# exception (so it can be seen if we're running with appropriate
# verbosity) and exit the argparse application with exit code 1 and
# a helpful error message.
LOGGER.exception('Could not load datastack')
parser.exit(DEFAULT_EXIT_CODE,
'Could not load datastack: %s\n' % str(error))
if args.workspace:
model_form.workspace.set_value(args.workspace)
# Run the UI's event loop
model_form.run(quickrun=args.quickrun)
app_exitcode = inputs.QT_APP.exec_()
# Handle a graceful exit
if model_form.form.run_dialog.messageArea.error:
parser.exit(DEFAULT_EXIT_CODE,
'Model %s: run failed\n' % args.model)
if app_exitcode != 0:
parser.exit(app_exitcode,
'App terminated with exit code %s\n' % app_exitcode)