def main(base, scen, mode, local, config_file):
logger = ReportService('default_log')
start_time = time.time()
wf = workflow(base, scen, mode, local, config_file)
steps = list(wf.keys())
for name, step in wf.items():
step.execute()
if name == 'ConfigureExecution':
logger = step.logger
logger.info('----------------STMA WorkFlow------------------')
for i, s in enumerate(steps):
logger.info("Step {:d} - {:30s}".format(i + 1, s))
end_time = time.time()
execution_time = max((end_time - start_time) / 60.0, 0.0)
model_status = [s.state for s in wf.values()]
model_status = {s: st for s, st in zip(steps, model_status)}
logger.info('----------------STMA Execution Summary------------------')
stma_status = TaskStatus.OK
stepid = 0
for step, status in model_status.items():
stepid += 1
logger.info('Step {:d} {:50s} Status = {:10s}'.format(
stepid, step, str(status)))
if status != TaskStatus.OK:
stma_status = TaskStatus.FAIL
if stma_status == TaskStatus.OK:
logger.info('STMA Completed in {:.0f} minutes'.format(execution_time))
else:
logger.error('STMA Failed in {:.0f} minutes'.format(execution_time))
return stma_status
class ControlService(object):
common_keys = ('TITLE', 'REPORT_FILE', 'PROJECT_DIRECTORY', 'RANDOM_SEED')
required_keys = ()
optional_keys = ()
def __init__(self, name=None, input_control_file=None):
self.name = name
self.title = None
self.exec_dir = os.getcwd()
self.control_file = input_control_file
self.control_files = []
self.keys = {}
self.tokens = {}
self.unused_keys = []
self.highest_group = 0
self.state = State.OK
self.project_dir = None
self.report_file = None
self.logger = None
self.seed = 42
@staticmethod
def is_comment(line):
return line.startswith("##") | line.startswith("//")
@staticmethod
def strip_comment(line):
"""
strip the comment
:param line: a string with comments at the end. Comments symbols not at the beginning
:return:
"""
loc_pound_sign = line.find("##")
loc_slash_sign = line.find("//")
if loc_pound_sign >= 0:
return line[:loc_pound_sign]
elif loc_slash_sign >= 0:
return line[:loc_slash_sign]
else:
return line
@staticmethod
def split_key_value(line):
"""
Replace all tabs with 4 spaces
:param line:
:return:
"""
key_value_pair = re.split(r"[\s]{2,}", line.replace('\t', ' '))
if len(key_value_pair) > 1:
return key_value_pair[0], key_value_pair[1]
else:
return key_value_pair[0], None
def replace_tokens(self, s, token='%'):
s = s.strip()
substrs = []
token_count, prev = 0, 0
for i, ch in enumerate(s):
if ch == token:
token_count += 1
if token_count == 1:
if i > 0:
substrs.append(s[prev:i])
prev = i
if token_count == 2:
substrs.append(s[prev:i + 1])
token_count = 0
prev = i + 1
if i == len(s) - 1:
substrs.append(s[prev:i + 1])
output = [s for s in substrs if s]
if token == '%':
for i, substr in enumerate(substrs):
if fnmatch(substr, '%*%'):
output[i] = os.environ.get(substr[1:-1], '').strip()
if token == '@':
for i, substr in enumerate(substrs):
if fnmatch(substr, '@*@'):
output[i] = self.tokens.get(substr, '')
return ''.join(output)
def read_control(self, control_file):
control_file = control_file.strip()
if self.state == State.ERROR:
return self.state
elif not os.path.exists(control_file):
self.state = State.ERROR
sys.stderr.write("Control file %s is not found\n" % control_file)
return self.state
else:
with open(control_file, mode='r') as control:
for line in control:
line = line.strip()
if line and not self.is_comment(line):
key, value = self.split_key_value(
self.strip_comment(line))
root, _ = self.get_root_key(key)
if root not in KEYS_DATABASE:
self.unused_keys.append(key)
else:
if value:
key = key.upper()
if key.startswith('CONTROL_KEY_FILE'):
# for case: CONTROL_KEY_FILE @SOME_FILE_NAME@
if fnmatch(value, "*@*@*"):
value = self.replace_tokens(value,
token='@')
elif fnmatch(value, "*%*%*"):
value = self.replace_tokens(value,
token='%')
value = os.path.join(
self.exec_dir, value
) # CONTROL_KEY_FILE relative to exec_dir
self.read_control(value)
elif fnmatch(key, "*@*@*"):
self.tokens[
key] = value # Collect program tokens @*@
else:
if fnmatch(value, "*%*%*"):
value = self.replace_tokens(value,
token='%')
self.keys[key] = Key(key=key,
input_value=value)
if self.keys[
key].group > self.highest_group:
self.highest_group = self.keys[
key].group
else:
self.keys[key] = Key(key=key)
def update_system_keys(self):
"""
Update the system keys
:return:
"""
if self.state == State.ERROR:
return self.state
if 'TITLE' not in self.keys:
self.keys['TITLE'] = Key('TITLE', input_value='')
if self.keys['TITLE'].input_value:
self.title = self.keys['TITLE'].input_value
else:
self.title = self.name
self.keys['TITLE'].value = self.title
if 'PROJECT_DIRECTORY' not in self.keys or self.keys[
'PROJECT_DIRECTORY'].input_value is None:
self.keys['PROJECT_DIRECTORY'] = Key('PROJECT_DIRECTORY')
self.project_dir = self.keys['PROJECT_DIRECTORY'].value
if 'REPORT_FILE' not in self.keys or not self.keys[
'REPORT_FILE'].input_value:
self.keys['REPORT_FILE'] = Key(
'REPORT_FILE',
input_value=os.path.basename(self.control_file)[:-4] + '.prn')
else:
if self.keys['REPORT_FILE'].input_value.find('.prn') < 0:
self.keys['REPORT_FILE'].value = self.keys[
'REPORT_FILE'].input_value + ".prn"
self.report_file = os.path.join(self.project_dir,
self.keys['REPORT_FILE'].value)
if 'RANDOM_SEED' not in self.keys or not self.keys[
'RANDOM_SEED'].input_value:
self.keys['RANDOM_SEED'] = Key('RANDOM_SEED')
self.logger = ReportService(self.report_file).get_logger()
@staticmethod
def parse_integer_list_key(value):
"""
Parse a range key
:param value: a string like 1,2,4..5
:return: a list with all individual values
"""
res = []
if ',' in value:
value_split = value.strip().split(',')
for v in value_split:
v_split = v.split('..')
if len(v_split) == 1:
res.append(int(v_split[0]))
else:
lb, hb = int(v_split[0]), int(v_split[1])
for k in range(lb, hb + 1):
res.append(k)
else:
v_split = value.split('..')
lb, hb = int(v_split[0]), int(v_split[1])
for k in range(lb, hb + 1):
res.append(k)
return res
@staticmethod
def parse_float_list_key(value):
"""
:param value:
:return:
"""
res = []
if ',' in value:
value_split = value.strip().split(',')
res = [float(v) for v in value_split]
else:
res.append(float(value))
return res
@staticmethod
def parse_boolean_key(value):
if str(value).upper().find('FALSE') >= 0 or str(value).upper() == '0':
return False
return True
def parse_time_range(self, time_range):
"""
parse the time range
:param time_range: parse comma-separated ranges like "0..6, 15..19"
:type time_range: str
:return: "0..6, 15..19" is parsed into [(0, 6), (15, 19)]
"""
if not isinstance(time_range, str):
self.logger.error("Time range must be a string")
return [(-1, -1)]
parts = time_range.split(",")
ranges = []
for part in parts:
if part.find("..") < 0:
self.logger.error(
"Time range must have both start and end times. Input is %s"
% part)
return [(-1, -1)]
else:
start_time, end_time = part.split("..")
ranges.append((float(start_time), float(end_time)))
return ranges
@staticmethod
def get_root_key(key_name):
"""
get the root key name
:param key_name:
:type key_name: str
:return:
"""
parts = key_name.split("_")
if parts[-1].isdigit():
return key_name[:len(key_name) - len(parts[-1]) - 1], int(
parts[-1])
else:
return key_name, 0
@staticmethod
def is_output_file(key_name):
if key_name.find('NEW') >= 0 and key_name.find('_FILE') >= 0:
return True
return False
def update_key_value(self, key):
if self.state == State.ERROR:
return self.state
if key.input_value is not None:
while key.input_value is not None and fnmatch(
key.input_value, "@*@"):
key.input_value = self.tokens.get(key.input_value)
while key.input_value is not None and fnmatch(
key.input_value, "%*%"):
key.input_value = self.tokens.get(key.input_value[1:-1])
if key.value_type == KeyValueTypes.TIME_RANGE:
val = self.parse_time_range(key.input_value)
if val[0][0] >= 0:
key.value = val
else:
self.state = State.ERROR
elif key.value_type == KeyValueTypes.STRING:
converter = str
key.value = converter(key.input_value)
elif key.value_type == KeyValueTypes.FLOAT:
converter = float
key.value = converter(key.input_value)
elif key.value_type == KeyValueTypes.INTEGER:
converter = int
key.value = converter(key.input_value)
elif key.value_type == KeyValueTypes.FILE:
if key.order > Offset.NETWORK_KEYS_OFFSET and self.project_dir and key.input_value:
key.value = os.path.join(self.project_dir, key.input_value)
elif key.value_type == KeyValueTypes.INTEGER_LIST:
key.value = self.parse_integer_list_key(key.input_value)
elif key.value_type == KeyValueTypes.FLOAT_LIST:
key.value = self.parse_float_list_key(key.input_value)
elif key.value_type == KeyValueTypes.BOOLEAN:
key.value = self.parse_boolean_key(key.input_value)
def check_keys(self):
"""
Populate all keys and check required key;
:return:
The key dictionary has all the acceptable keys in fully suffixed notation; If the value is None, the key
is not set by the user
"""
if self.state == State.ERROR:
return self.state
acceptable_keys = self.common_keys + self.required_keys + self.optional_keys
single_keys = [
k for k in acceptable_keys
if KEYS_DATABASE[k].group == KeyGroupTypes.SINGLE
]
group_suffixes = [
"_" + str(g) for g in range(1, self.highest_group + 1)
]
group_keys = [
k for k in acceptable_keys
if KEYS_DATABASE[k].group == KeyGroupTypes.GROUP
]
full_key_list = single_keys + [
k + s for k, s in itertools.product(group_keys, group_suffixes)
]
for k in full_key_list:
if k not in self.keys:
root_key_name, key_group = self.get_root_key(k)
key_value = KEYS_DATABASE[root_key_name].default
if key_group > 1:
# take the value of the first defined group if not specified; otherwise default value
prev_group = key_group - 1
while root_key_name + "_" + str(
prev_group) not in self.keys:
prev_group -= 1
key_value = self.keys[root_key_name + "_" +
str(prev_group)].input_value
new_key = Key(key=k, input_value=key_value)
self.keys.update({k: new_key})
# Update the key value to internal data structures
for k in self.keys.values():
self.update_key_value(k)
# Check required keys for each group. if a value is None, raise an error
check_key = None
# found = False
for req_k in self.required_keys:
found = False
is_group_key = KEYS_DATABASE[req_k].group == KeyGroupTypes.GROUP
if is_group_key:
for g in group_suffixes:
check_key = req_k + g
for name, k in self.keys.items():
if check_key == name and k.input_value is not None and len(
k.input_value) > 0:
found = True
break
else:
check_key = req_k
for name, k in self.keys.items():
if check_key == name and k.value is not None:
found = True
break
if not found:
self.logger.error("Required key %s not found" % check_key)
self.state = State.ERROR
def check_files(self):
if self.state == State.ERROR:
return self.state
for k in self.keys.values():
if k.key == 'PROJECT_DIRECTORY':
if not os.path.exists(k.value):
self.state = State.ERROR
self.logger.error("Project Directory %s does not exist" %
k.value)
if k.value_type == KeyValueTypes.FILE:
if self.is_output_file(k.key):
if k.value and not os.path.exists(os.path.dirname(
k.value)):
self.state = State.ERROR
self.logger.error("Path %s for %s does not exist" %
(os.path.dirname(k.value), k.key))
elif k.key != 'REPORT_FILE':
if k.value and not os.path.exists(k.value):
self.state = State.ERROR
self.logger.error("File %s for %s does not exist" %
(k.value, k.key))
def print_keys(self):
if self.state == State.ERROR:
return self.state
keys = [(k, v.value, v.order) for k, v in self.keys.items()]
keys = sorted(keys, key=lambda k: k[2])
for k, v, _ in keys:
self.logger.info("%s = %s" % (k, v))
for key in self.unused_keys:
self.logger.warning("Unused key {:s}".format(key))
self.logger.info("")
self.logger.info("")
def execute(self):
self.read_control(self.control_file)
self.update_system_keys()
self.check_keys()
self.print_keys()
self.check_files()
return self.state
class ConfigureExecution(Task):
family = 'Setup'
def __init__(self, base, scen, mode, config_file, local_network='FALSE', step_id='00'):
super().__init__(step_id=step_id)
self.base = base
self.scen = scen
self.mode = mode.upper()
self.local_network = local_network
if local_network.upper() == 'TRUE':
self.local_network = True
else:
self.local_network = False
self.config_file = config_file
self.swift_dir = None
self.stma_software_dir = None
self.dynust_dir = None
self.dynastudio_executable = None
self.dynust_executable_name = None
self.common_dir = None
self.base_dir = None
self.scen_dir = None
self.config = None
self.threads = 1
self.logger = None
def check_dir(self, d, root_dir=None):
if not os.path.exists(d):
os.mkdir(d)
if not os.path.exists(d):
if root_dir is not None:
self.logger.error('{:60s} Failed to Create'.format(os.path.relpath(d, root_dir)))
else:
self.logger.error('{:60s} Failed to Create'.format(d))
return False
else:
if root_dir is not None:
self.logger.info('{:60s} Created'.format(os.path.relpath(d, root_dir)))
else:
self.logger.info('{:60s} Created'.format(d))
return True
else:
if root_dir is not None:
self.logger.info('{:60s} Checked'.format(os.path.relpath(d, root_dir)))
else:
self.logger.info('{:60s} Checked'.format(d))
return True
def require(self):
current_dir = os.getcwd()
if self.config_file.startswith('.'):
self.config_file = os.path.join(current_dir, self.config_file)
if not os.path.exists(self.config_file):
self.state = State.ERROR
sys.stderr('STM-A Configuration File {:s} Not Found - STM-A Failed'.format(self.config_file))
return self.state
def run(self):
if self.state == TaskStatus.OK:
parser = configparser.ConfigParser()
parser.read(self.config_file)
self.swift_dir = parser['SYSTEM']['SWIFT_Directory']
self.stma_software_dir = parser['SYSTEM']['STMA_Software_Directory']
self.dynust_dir = parser['SYSTEM']['DynusT_Software_Directory']
self.dynastudio_executable = parser['SYSTEM']['DynuStudio_Executable']
self.dynust_executable_name = parser['SYSTEM']['DynusT_Executable_Name']
self.common_dir = os.path.join(self.swift_dir, 'CommonData')
self.threads = int(parser['SYSTEM']['Number_Threads'])
self.scen_dir = os.path.join(self.swift_dir, 'Scenarios', self.scen)
self.logger = ReportService(os.path.join(self.scen_dir, self.scen + '_STM_A.log')).get_logger()
if not self.check_dir(self.scen_dir):
self.logger.error('Scenario {:s} Not Found at'.format(self.scen_dir))
self.state = State.ERROR
return self.state
self.base_dir = os.path.join(self.swift_dir, 'Scenarios', self.base)
if not os.path.exists(self.base_dir):
self.logger.error('Baseline {:s} Not Found at'.format(self.base_dir))
self.state = State.ERROR
return self.state
if self.mode.upper() == 'FULL':
self.mode = "FULL"
else:
self.mode = 'QUICK'
self.logger.info('Scenario {:s} Execution Starts'.format(self.scen))
self.logger.info('')
self.logger.info('TASK {:s}_{:s}_{:s}: START'.format(self.family, self.step_id, self.__class__.__name__))
self.logger.info('')
self.logger.info('Scenario Name = {:s}'.format(self.scen))
self.logger.info('Baseline Name = {:s}'.format(self.base))
self.logger.info('Execution Mode = {:s}'.format(self.mode))
self.logger.info('Use Local Network = {:}'.format(self.local_network))
self.logger.info('Configuration File = {:s}'.format(self.config_file))
self.logger.info('Number of Threads = {:d}'.format(self.threads))
def complete(self):
message = 'TASK {:s}_{:s}_{:s}: STATUS = {:15s}'.format(
self.family, self.step_id, self.__class__.__name__, str(self.state))
self.logger.info('')
self.logger.info('')
self.logger.info(message)
self.logger.info('')
self.logger.info('')
def execute(self):
self.require()
self.run()
self.complete()
return self.state