class ApplicationsSetup(object):
"""Manage applications that make up the master run workflow."""
def __init__(
self,
aCmdLineParameters,
aSysArgs,
aConfigPath=None,
aSingleRun=False,
aAddOpts=False,
):
"""Initialize ApplicationSetup object."""
lsf_info = self._getAppConfig("lsf")
self._mAppsInfo = ApplicationsInfo(lsf_info)
# Setting the config path for access in master run (so we can pass it
# to forrest run)
self._mAppsInfo.mConfigPath = aConfigPath
# Getting workflow lists
(single_run_app_opts, sequence_app_opts) = self._getWorkflow(aConfigPath)
# Populate single run applications options
if aSingleRun:
for app_name, app_opts in single_run_app_opts:
app_info = self._getAppConfig(app_name)
self._mAppsInfo.addSingleRunApplication(app_info)
self._addWorkflowToSysArgs(aSysArgs, app_name, app_opts)
# Populate sequence applications options
for app_name, app_opts in sequence_app_opts:
app_info = self._getAppConfig(app_name)
self._mAppsInfo.addSequenceApplication(app_info)
if aSingleRun: # We don't need to add these to forrest run's command line
self._addWorkflowToSysArgs(aSysArgs, app_name, app_opts)
# Add options from applications if necessary
if aAddOpts:
self._addAppOptions(aCmdLineParameters)
self._mAppsInfo.mCmdLineOpts = CmdLineOpts(aCmdLineParameters, aSysArgs)
# Let applications that have added parameters receive the parsed
# result.
if aAddOpts:
self._resolveParameters()
# Adds options to provided system arguments to resolve later
#
def _addWorkflowToSysArgs(self, aSysArgs, aAppName, aAppOpts):
"""Adds options to provided system arguments to resolve later"""
for option, value in aAppOpts.items():
argument = "--%s.%s" % (aAppName, option)
if argument in aSysArgs:
if value is not None:
option_index = aSysArgs.index(argument) + 1
new_argument = [aSysArgs[option_index]]
# Add new argument to existing options
if isinstance(value, list):
new_argument.extend(value)
aSysArgs[option_index] = new_argument
else:
new_argument.append(value)
aSysArgs[option_index] = new_argument
else: # Argument does not exist yet
aSysArgs.append(argument)
if value is not None:
aSysArgs.append(value)
# Retrieve workflow from the provided path to add to aSysArgs to parse
# when building mCmdLineOpts later
#
def _getWorkflow(self, aConfigPath):
"""Retrieve workflow from the provided path to add to aSysArgs to
parse when building mCmdLineOpts later"""
# Check workflow in config file first
try:
Msg.info("Using workflow from: %s" % aConfigPath)
config_module = SourceFileLoader("config", aConfigPath).load_module()
return (
config_module.single_run_app_opts,
config_module.sequence_app_opts,
)
except AttributeError:
# aConfigPath is None or the specified config file does not
# contain workflow
if aConfigPath:
Msg.info("Workflow improperly defined in %s" % aConfigPath)
else:
Msg.info("Config not specified.")
# Check environment variable next
try:
Msg.info("Attempting to use MASTER_RUN_CONFIG environment variable.")
config_module = SourceFileLoader(
"config", os.environ.get("MASTER_RUN_CONFIG")
).load_module()
return (
config_module.single_run_app_opts,
config_module.sequence_app_opts,
)
except AttributeError:
if os.environ.get("MASTER_RUN_CONFIG"):
Msg.info(
"Workflow improperly defined in MASTER_RUN_CONFIG: "
"%s" % os.environ.get("MASTER_RUN_CONFIG")
)
else:
Msg.info("MASTER_RUN_CONFIG environment variable is not set.")
except FileNotFoundError: # MASTER_RUN_CONFIG environment variable
# is set, but cannot be found
Msg.err(
"MASTER_RUN_CONFIG is currently set to %s. "
"Please ensure that it exists." % os.environ.get("MASTER_RUN_CONFIG")
)
sys.exit(1) # Assume typo so quit
# Use default last
try:
default_config_file = "%s/config/%s" % (
sys.path[0],
Defaults.fcfg_name,
)
Msg.info("Using workflow from default config file: %s" % default_config_file)
config_module = SourceFileLoader("config", default_config_file).load_module()
return (
config_module.single_run_app_opts,
config_module.sequence_app_opts,
)
except FileNotFoundError: # default config file cannot be found
Msg.err("Please ensure the default config file exists.")
sys.exit(1) # Assume typo so quit
# Return the ApplicationsInfo object
#
def getApplicationsInfo(self):
"""Return the ApplicationsInfo object"""
return self._mAppsInfo
# Import and create instance of ApplicationConfig for the requested
# application
#
def _getAppConfig(self, aAppName):
"""Import and create instance of ApplicationConfig for the requested
application"""
app_module = importlib.import_module("." + aAppName, "applications")
app_cfg = ApplicationConfig(aAppName, app_module)
return app_cfg
# Iterate through all ApplicationConfig objects to add application
# specific command line parameters
#
def _addAppOptions(self, aCmdLineParameters):
"""Iterate through all ApplicationConfig objects to add application
specific command line parameters"""
for app_cfg in self._mAppsInfo.mAllAppsOrder:
app_cmdline_opts = app_cfg.getCmdLineOptions()
if app_cmdline_opts is not None:
aCmdLineParameters.group_names.append(app_cmdline_opts.cGroupName)
aCmdLineParameters.group_descriptions.append(app_cmdline_opts.cGroupDescription)
app_options_expanded = list()
for app_option in app_cmdline_opts.cOptions:
app_option.mAppName = app_cfg.mName
app_options_expanded.append(app_option.getExpandedList())
aCmdLineParameters.group_parameters.append(app_options_expanded)
aCmdLineParameters.parameters.extend(app_options_expanded)
# Iterate through all ApplicationConfig objects to pass back parsed
# command line options and resolve application parameters.
#
def _resolveParameters(self):
"""Iterate through all ApplicationConfig objects to pass back parsed
command line options and resolve application parameters."""
for app_cfg in self._mAppsInfo.mAllAppsOrder:
app_parms_processor_cls = app_cfg.getParametersProcessorClass()
if app_parms_processor_cls is not None:
app_parms_processor = app_parms_processor_cls(self._mAppsInfo.mCmdLineOpts)
app_cfg.mAppParameters = app_parms_processor.mAppParameters
class ApplicationsSetup(object):
def __init__(self,
aCmdLineParameters,
aSysArgs,
aConfigPath=None,
aSingleRun=False,
aAddOpts=False):
lsf_info = self._getAppConfig('lsf')
self._mAppsInfo = ApplicationsInfo(lsf_info)
# Setting the config path for access in master run (so we can pass it to forrest run)
self._mAppsInfo.mConfigPath = aConfigPath
# Getting workflow lists
(single_run_app_opts,
sequence_app_opts) = self._getWorkflow(aConfigPath)
# Populate single run applications options
if aSingleRun:
# TODO: The output directory hasn't been initialized when we get to this point, so we aren't able to determine if the workflow file should be written
# to output/regression/ or output/performance/ yet. Ideally we would set up all of master run before doing any application setup, but the current
# implementation is the other way around. Fixing this issue is too large an effort for something this small, but this should be revisited when 1) we
# decide to knock out some of the major technical debt that master run has accrued, 2) we decide on being able to include workflow in control files,
# or 3) we decide to start explicitly logging workflow files. For now, workflow is being passed into forrest run via the config file.
#self._mAppsInfo.mConfigPath = self._buildTemporaryWorkflowFile(single_run_app_opts, sequence_app_opts)
for app_name, app_opts in single_run_app_opts:
app_info = self._getAppConfig(app_name)
self._mAppsInfo.addSingleRunApplication(app_info)
self._addWorkflowToSysArgs(aSysArgs, app_name, app_opts)
# Populate sequence applications options
for app_name, app_opts in sequence_app_opts:
app_info = self._getAppConfig(app_name)
self._mAppsInfo.addSequenceApplication(app_info)
if aSingleRun: # We don't need to add these to forrest run's command line
self._addWorkflowToSysArgs(aSysArgs, app_name, app_opts)
# Add options from applications if necessary
if aAddOpts:
self._addAppOptions(aCmdLineParameters)
self._mAppsInfo.mCmdLineOpts = CmdLineOpts(aCmdLineParameters,
aSysArgs)
# Let applications that have added parameters receive the parsed result.
if aAddOpts:
self._resolveParameters()
## Adds options to provided system arguments to resolve later
#
def _addWorkflowToSysArgs(
self, aSysArgs, aAppName, aAppOpts
): # TODO: Currently, the workflow file is arranged as a list of tuples: (application name, list of
for option, value in aAppOpts.items(
): # dictionaries associated with the app). The way workflow parsing works right now is to split
argument = '--%s.%s' % (
aAppName, option
) # each dictionary into a key and value and append them to existing lists of values for the
if argument in aSysArgs: # associated key. It is the application's responsibility to handle the list of options for each
if value is not None: # key. Since this feature was originally intended for use with the Iss application and Iss has
option_index = aSysArgs.index(
argument
) + 1 # only 1 option (path), it works perfectly. Issues arise when this functionality is adapted for
new_argument = [
aSysArgs[option_index]
] # future applications which may have more than 1 optional option: if the first dictionary for
# Add new argument to existing options # app X contained 2 key-value pairs (a:'a', b:'b') and the second contained a single (a:'a2'),
if isinstance(
value, list
): # the application would get 2 lists of different sizes (a:['a', 'a2'], b:['b']) and would be
new_argument.extend(
value
) # unable to determine which a-value the single b-value maps to.
aSysArgs[option_index] = new_argument #
else: # There are 2 clear solutions available. The first is to change master run's behavior: how it handles
new_argument.append(
value
) # options, and loading them into their respective applications. Doing this would not require each
aSysArgs[
option_index] = new_argument # application having its own custom handling of multiple dictionaries, however would change some core
else: # Argument does not exist yet # functionality of master run. The second is to modify this current workflow parser to pass entire
aSysArgs.append(
argument
) # dictionaries into applications instead of a list for each option. This does not change any major
if value is not None: # master run functionality but requires custom handling in each application.
aSysArgs.append(value)
## Retrieve workflow from the provided path to add to aSysArgs to parse when building mCmdLineOpts later
#
def _getWorkflow(self, aConfigPath):
# Check workflow in config file first
try:
Msg.info('Using workflow from: %s' % aConfigPath)
config_module = SourceFileLoader('config',
aConfigPath).load_module()
return (config_module.single_run_app_opts,
config_module.sequence_app_opts)
except AttributeError: # aConfigPath is None or the specified config file does not contain workflow
if aConfigPath:
Msg.info('Workflow improperly defined in %s' % aConfigPath)
else:
Msg.info('Config not specified.')
# Check environment variable next
try:
Msg.info(
'Attempting to use MASTER_RUN_CONFIG environment variable.')
config_module = SourceFileLoader(
'config', os.environ.get('MASTER_RUN_CONFIG')).load_module()
return (config_module.single_run_app_opts,
config_module.sequence_app_opts)
except AttributeError:
if os.environ.get('MASTER_RUN_CONFIG'):
Msg.info(
'Workflow improperly defined in MASTER_RUN_CONFIG: %s' %
os.environ.get('MASTER_RUN_CONFIG'))
else:
Msg.info('MASTER_RUN_CONFIG environment variable is not set.')
except FileNotFoundError: # MASTER_RUN_CONFIG environment variable is set, but cannot be found
Msg.err(
'MASTER_RUN_CONFIG is currently set to %s. Please ensure that it exists.'
% os.environ.get('MASTER_RUN_CONFIG'))
sys.exit(1) # Assume typo so quit
# Use default last
try:
default_config_file = '%s/config/%s' % (sys.path[0],
Defaults.fcfg_name)
Msg.info('Using workflow from default config file: %s' %
default_config_file)
config_module = SourceFileLoader(
'config', default_config_file).load_module()
return (config_module.single_run_app_opts,
config_module.sequence_app_opts)
except FileNotFoundError: # default config file cannot be found
Msg.err('Please ensure the default config file exists.')
sys.exit(1) # Assume typo so quit
## Return the ApplicationsInfo object
#
def getApplicationsInfo(self):
return self._mAppsInfo
## Import and create instance of ApplicationConfig for the requested application
#
def _getAppConfig(self, aAppName):
app_module = importlib.import_module('.' + aAppName, 'applications')
app_cfg = ApplicationConfig(aAppName, app_module)
return app_cfg
## Iterate through all ApplicationConfig objects to add application specific command line parameters
#
def _addAppOptions(self, aCmdLineParameters):
for app_cfg in self._mAppsInfo.mAllAppsOrder:
app_cmdline_opts = app_cfg.getCmdLineOptions()
if app_cmdline_opts is not None:
aCmdLineParameters.group_names.append(
app_cmdline_opts.cGroupName)
aCmdLineParameters.group_descriptions.append(
app_cmdline_opts.cGroupDescription)
app_options_expanded = list()
for app_option in app_cmdline_opts.cOptions:
app_option.mAppName = app_cfg._mName
app_options_expanded.append(app_option.getExpandedList())
aCmdLineParameters.group_parameters.append(
app_options_expanded)
aCmdLineParameters.parameters.extend(app_options_expanded)
## Iterate through all ApplicationConfig objects to pass back parsed command line options and resolve application parameters.
#
def _resolveParameters(self):
for app_cfg in self._mAppsInfo.mAllAppsOrder:
app_parms_processor_cls = app_cfg.getParametersProcessorClass()
if app_parms_processor_cls is not None:
app_parms_processor = app_parms_processor_cls(
self._mAppsInfo.mCmdLineOpts)
app_cfg.mAppParameters = app_parms_processor.mAppParameters