def read_externals_description_file(root_dir, file_name):
"""Given a file name containing a externals description, determine the
format and read it into it's internal representation.
"""
root_dir = os.path.abspath(root_dir)
msg = 'In directory : {0}'.format(root_dir)
logging.info(msg)
printlog('Processing externals description file : {0}'.format(file_name))
file_path = os.path.join(root_dir, file_name)
if not os.path.exists(file_name):
msg = ('ERROR: Model description file, "{0}", does not '
'exist at path:\n {1}\nDid you run from the root of '
'the source tree?'.format(file_name, file_path))
fatal_error(msg)
externals_description = None
try:
config = config_parser()
config.read(file_path)
externals_description = config
except MissingSectionHeaderError:
# not a cfg file
pass
if externals_description is None:
msg = 'Unknown file format!'
fatal_error(msg)
return externals_description
def __init__(self):
"""
Constructor for Utilities class
"""
# Initiate Config Parser instance
self.parser = config_parser()
# Read config file inside
self.parser.read(CONFIG_FILE)
# Set Config object by config parser sections
CONFIG.update(self.parser._sections)
# Set global NOW value to be used in any where in application
self.now = strftime(self.config_get("locale", "datetime_format"))
# Set log destination
# Convert Json string into DotDict/OrderedDict object
log_destination = json.loads(unicode(self.config_get("logging", "destination")))
# Initiate Syslog handler with log destination regarding to the system architecture
syslog = SysLogHandler(address=log_destination[sys.platform])
# Set syslog format
syslog.setFormatter(
logging.Formatter(self.config_get("logging", "format"))
)
self.logger.addHandler(syslog)
def __init__(self):
"""
Constructor for Utilities class
"""
# Initiate Config Parser instance
self.parser = config_parser()
# Read config file inside
self.parser.read(CONFIG_FILE)
# Set Config object by config parser sections
CONFIG.update(self.parser._sections)
# Set global NOW value to be used in any where in application
self.now = strftime(self.config_get("locale", "datetime_format"))
# Set log destination
# Convert Json string into DotDict/OrderedDict object
log_destination = json.loads(
unicode(self.config_get("logging", "destination")))
# Initiate Syslog handler with log destination regarding to the system architecture
syslog = SysLogHandler(address=log_destination[sys.platform])
# Set syslog format
syslog.setFormatter(
logging.Formatter(self.config_get("logging", "format")))
self.logger.addHandler(syslog)
def write_config_template(filename):
"""
"""
print(filename)
template = config_parser()
section = VERSION_CONTROL
template.add_section(section)
template.set(section, GIT_REF_REPO, 'relative path to main repository')
template.set(
section, BRANCH_BASE,
'"base name of branch. changeset will be appended. '
'example foo-bar becomes foo-bar-ID."')
template.set(section, CHANGESET_IDS, 'comma separeted list of '
'changeset ids')
section = BISECT_TEST
template.add_section(section)
template.set(section, BISECT_TYPE, '"test" or "newcase"')
template.set(section, RUN_TEST, 'only if "test" type, e.g. ERS_D_Ld3')
template.set(section, RESOLUTION, 'e.g. f10_f10')
template.set(section, COMPSET, 'e.g. ICLM50BGC')
template.set(section, MACHINE, 'supported machine name')
template.set(section, COMPILER, 'supported compiler name')
template.set(section, TEST_MODS, 'only if "test" type, e.g. clm-default')
template.set(
section, TEST_COMMANDS,
'comma separated list of quoted strings with commands needed to run the test.'
)
with open('{0}.cfg'.format(filename), 'wb') as configfile:
template.write(configfile)
def determine_test_info(test_info_file):
"""Determine what machine we are running on and extract the test
config written by test-clm
"""
print("Checking test info.")
config = config_parser()
config.read(test_info_file)
test_info = {}
for section in config.sections():
for option in config.options(section):
test_info[option] = config.get(section, option)
print("Using test info :")
for key in test_info:
print(" {0} : {1}".format(key, test_info[key]))
if "cesm_src_dir" not in test_info:
raise RuntimeError("cesm_src_dir key not in test config file. Please update test config file.")
config_machines_xml = "{0}/scripts/ccsm_utils/Machines/config_machines.xml".format(test_info["cesm_src_dir"])
machine, machine_config = read_machine_config(None, config_machines_xml)
check_test_info(machine_config, test_info)
return machine, test_info
def read_externals_description_file(root_dir, file_name):
"""Given a file name containing a externals description, determine the
format and read it into it's internal representation.
"""
root_dir = os.path.abspath(root_dir)
msg = 'In directory : {0}'.format(root_dir)
logging.info(msg)
printlog('Processing externals description file : {0}'.format(file_name))
file_path = os.path.join(root_dir, file_name)
if not os.path.exists(file_name):
msg = ('ERROR: Model description file, "{0}", does not '
'exist at path:\n {1}\nDid you run from the root of '
'the source tree?'.format(file_name, file_path))
fatal_error(msg)
externals_description = None
try:
config = config_parser()
config.read(file_path)
externals_description = config
except MissingSectionHeaderError:
# not a cfg file
pass
if externals_description is None:
msg = 'Unknown file format!'
fatal_error(msg)
return externals_description
def __init__(self, filename, defaults ={}, section="settings"):
ini = config_parser()
if os.path.isfile(filename):
ini.read(filename)
self.ini = ini
self.defaults = defaults
self.section = section
def determine_test_info(test_info_file):
"""Determine what machine we are running on and extract the test
config written by test-clm
"""
print("Checking test info.")
config = config_parser()
config.read(test_info_file)
test_info = {}
for section in config.sections():
for option in config.options(section):
test_info[option] = config.get(section, option)
print("Using test info :")
for key in test_info:
print(" {0} : {1}".format(key, test_info[key]))
if 'cesm_src_dir' not in test_info:
raise RuntimeError("cesm_src_dir key not in test config file. Please update test config file.")
config_machines_xml = "{0}/scripts/ccsm_utils/Machines/config_machines.xml".format(test_info['cesm_src_dir'])
machine, machine_config = read_machine_config(None, config_machines_xml)
check_test_info(machine_config, test_info)
return machine, test_info
def setUp(self):
"""Reusable config object
"""
self._config = config_parser()
self._config.add_section('section1')
self._config.set('section1', 'keword', 'value')
self._config.add_section(DESCRIPTION_SECTION)
def test_one_tag_required(self):
"""Test that a component source with a tag is correctly parsed.
"""
config = config_parser()
self._setup_comp1(config)
model = ExternalsDescriptionConfigV1(config)
print(model)
self._check_comp1(model)
def test_one_branch_externals(self):
"""Test that a component source with a branch is correctly parsed.
"""
config = config_parser()
self._setup_comp2(config)
model = ExternalsDescriptionConfigV1(config)
print(model)
self._check_comp2(model)
def setUp(self):
"""Reusable config object
"""
self._config = config_parser()
self._config.add_section('section1')
self._config.set('section1', 'keword', 'value')
self._config.add_section(DESCRIPTION_SECTION)
def test_cfg_v1_reject_v1_too_new(self):
"""Test that a v1 description object won't try to parse a v2 file.
"""
config = config_parser()
self._setup_comp1(config)
self._setup_externals_description(config)
config.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.100.0')
with self.assertRaises(RuntimeError):
ExternalsDescriptionConfigV1(config)
def test_cfg_v1_reject_unknown_item(self):
"""Test that a v1 description object will reject unknown items
"""
config = config_parser()
self._setup_comp1(config)
self._setup_externals_description(config)
config.set(self._comp1_name, 'junk', 'foobar')
with self.assertRaises(RuntimeError):
ExternalsDescriptionConfigV1(config)
def test_cfg_v1_reject_v1_too_new(self):
"""Test that a v1 description object won't try to parse a v2 file.
"""
config = config_parser()
self._setup_comp1(config)
self._setup_externals_description(config)
config.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.100.0')
with self.assertRaises(RuntimeError):
ExternalsDescriptionConfigV1(config)
def test_one_branch_externals(self):
"""Test that a component source with a branch is correctly parsed.
"""
config = config_parser()
self._setup_comp2(config)
self._setup_externals_description(config)
model = ExternalsDescriptionConfigV1(config)
print(model)
self._check_comp2(model)
def test_cfg_v1_reject_unknown_item(self):
"""Test that a v1 description object will reject unknown items
"""
config = config_parser()
self._setup_comp1(config)
self._setup_externals_description(config)
config.set(self._comp1_name, 'junk', 'foobar')
with self.assertRaises(RuntimeError):
ExternalsDescriptionConfigV1(config)
def test_two_sources(self):
"""Test that multiple component sources are correctly parsed.
"""
config = config_parser()
self._setup_comp1(config)
self._setup_comp2(config)
model = ExternalsDescriptionConfigV1(config)
print(model)
self._check_comp1(model)
self._check_comp2(model)
def test_two_sources(self):
"""Test that multiple component sources are correctly parsed.
"""
config = config_parser()
self._setup_comp1(config)
self._setup_comp2(config)
self._setup_externals_description(config)
model = ExternalsDescriptionConfigV1(config)
print(model)
self._check_comp1(model)
self._check_comp2(model)
def read_config_file(filename):
"""Read the configuration file and process"""
print("tools_autodoc.py - Reading configuration file : {0}".format(filename))
cfg_file = os.path.abspath(filename)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find config file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
return config
def write_template_config():
"""write a template configuration file.
Each section of the configuration file is a different test suite.
Each section has a 'type' key that defines how it is processed.
- base : defineds a base simulation for perturbation suites
- single parameter perturbation : start with the base simulation
and perturb it once for each item in each key.
- one off : a simple list of tests that are included as is
The keys in each section correspond to the different parts of a
cime test name:
{test}.{grid}.{compset}.{machine}_{compiler}.{testmod}
The keys for "one off" are basically meaningless, and can be used
for grouping. The values are space delimited lists of fullly
qualified test names.
"""
template = config_parser()
section = 'clm_base'
template.add_section(section)
template.set(section, 'type', '"base"')
template.set(section, 'compset', 'string')
template.set(section, 'grid', 'string')
template.set(section, 'test', 'string')
template.set(section, 'testmods', 'string')
template.set(section, 'machine', 'string')
template.set(section, 'compiler', 'string')
section = 'clm_spp'
template.add_section(section)
template.set(section, 'type', '"single param perturbation"')
template.set(section, 'compset', 'space separated list')
template.set(section, 'grid', 'space separated list')
template.set(section, 'test', 'space separated list')
template.set(section, 'testmods', 'space separated list')
template.set(section, 'machine', 'string')
template.set(section, 'compiler', 'string')
section = 'clm_long'
template.add_section(section)
template.set(section, 'type', '"one off"')
template.set(section, 'tests', 'space separated list')
with open('template.cfg', 'wb') as configfile:
template.write(configfile)
def get_configs(f, sec):
config = config_parser(allow_no_value=True)
result = {}
try:
config.read(f)
if sec in config.sections():
for k, v in config.items(sec):
result[k] = v
return result
else:
return {}
except Exception:
return {}
def read_config(f):
config = config_parser(allow_no_value=True)
ctx = Context()
try:
config.read(f)
if 'environ' in config.sections():
for c in config.items('environ'):
os.environ[c[0]] = c[1]
os.environ['PATH'] += ';' + c[1]
ctx.set(c[0], c[1])
return True
return False
except Exception:
return False
def read_config_file(filename):
"""Read the configuration file and process
"""
print("Reading configuration file : {0}".format(filename))
cfg_file = os.path.abspath(filename)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find config file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
return config
def _read_cccg_config_file():
"""
"""
from ConfigParser import SafeConfigParser as config_parser
cccg_config_file = os.path.abspath(
os.path.join(os.path.expanduser("~"), ".cccg", "config"))
cccg_config = config_parser()
if (os.path.isfile(cccg_config_file)):
cccg_config.read(cccg_config_file)
else:
cccg_config.add_section('main')
return cccg_config
def read_config_file(filename):
"""Read the configuration file and process
"""
logging.info("Reading configuration file :")
logging.info(" {0}".format(filename))
cfg_file = os.path.abspath(filename)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
return config
def _read_cime_config_file():
"""
READ the config file in ~/.cime, this file may contain
[main]
CIMEROOT=/path/to/cime
CIME_MODEL=acme,cesm
PROJECT=someprojectnumber
"""
cime_config_file = os.path.abspath(os.path.join(os.path.expanduser("~"),
".cime","config"))
cime_config = config_parser()
if(os.path.isfile(cime_config_file)):
cime_config.read(cime_config_file)
else:
logger.debug("File %s not found" % cime_config_file)
cime_config.add_section('main')
return cime_config
def _read_cime_config_file():
"""
READ the config file in ~/.cime, this file may contain
[main]
CIME_MODEL=acme,cesm
PROJECT=someprojectnumber
"""
from ConfigParser import SafeConfigParser as config_parser
cime_config_file = os.path.abspath(
os.path.join(os.path.expanduser("~"), ".cime", "config"))
cime_config = config_parser()
if (os.path.isfile(cime_config_file)):
cime_config.read(cime_config_file)
else:
logger.debug("File %s not found" % cime_config_file)
cime_config.add_section('main')
return cime_config
def read_machine_config(cime_version, cfg_file, config_machines_xml):
"""Read the configuration file and convert machine info into a dict. Expected format:
[yellowstone]
host = yslogin
batch = execca
background = true|false
clm_compilers = intel, pgi
Note that we skip any section that doesn't have a 'host' keyword.
"""
print("Reading machine configuration file : {0}".format(cfg_file))
cfg_file = os.path.abspath(cfg_file)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find config file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
config_dict = {}
for section in config.sections():
if not config.has_option(section, 'host'):
continue
config_dict[section] = {}
for i in config.items(section):
key = i[0]
value = i[1]
config_dict[section][key] = value
machine = get_machine(config_dict)
machine_config = config_dict[machine]
print("{0} configuration :".format(machine))
for key in machine_config:
print(" {0} : {1}".format(key, machine_config[key]))
machine_xml_config = read_config_machines_xml(cime_version, machine,
config_machines_xml)
print("{0} xml :".format(machine))
for key in machine_xml_config:
print(" {0} : {1}".format(key, machine_xml_config[key]))
machine_config.update(machine_xml_config)
return machine, machine_config
def read_machine_config(cfg_file, config_machines_xml):
"""Read the configuration file and convert machine info into a dict. Expected format:
[yellowstone]
host = yslogin
batch = execca
background = true|false
clm_compilers = intel, pgi
Note that we skip any section that doesn't have a 'host' keyword.
"""
print("Reading machine configuration file : {0}".format(cfg_file))
cfg_file = os.path.abspath(cfg_file)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find config file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
config_dict = {}
for section in config.sections():
if not config.has_option(section, 'host'):
continue
config_dict[section] = {}
for i in config.items(section):
key = i[0]
value = i[1]
config_dict[section][key] = value
machine = get_machine(config_dict)
machine_config = config_dict[machine]
print("{0} configuration :".format(machine))
for key in machine_config:
print(" {0} : {1}".format(key, machine_config[key]))
machine_xml_config = read_config_machines_xml(machine, config_machines_xml)
print("{0} xml :".format(machine))
for key in machine_xml_config:
print(" {0} : {1}".format(key, machine_xml_config[key]))
machine_config.update(machine_xml_config)
return machine, machine_config
def read_suite_config(cfg_file, suite_name):
"""Read the configuration file and look for suite section. This
translates the testlist nonsence into simple test suites. Expected
format:
[suites]
suite_name = testlist_xml_name1, testlist_xml_name2
clm = aux_clm40, aux_clm45
"""
print("Reading configuration file : {0}".format(cfg_file))
cfg_file = os.path.abspath(cfg_file)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find config file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
section = 'suites'
if not config.has_section(section):
raise RuntimeError("ERROR: config file must contain a "
"'{0}' section.".format(section))
suites = {}
for option in config.options(section):
tmp = config.get(section, option)
suites[option] = tmp.split(',')
for s in suites:
suites[s] = [l.strip() for l in suites[s]]
print("Known test suites:")
for s in suites:
print(" {0} : {1}".format(s, ', '.join(suites[s])))
if suite_name not in suites:
raise RuntimeError("ERROR: config file does not contain a test suite '{0}'".format(suite_name))
return suites[suite_name]
def read_suite_config(cfg_file, suite_name):
"""Read the configuration file and look for suite section. This
translates the testlist nonsence into simple test suites. Expected
format:
[suites]
suite_name = testlist_xml_name1, testlist_xml_name2
clm = aux_clm40, aux_clm45
"""
print("Reading configuration file : {0}".format(cfg_file))
cfg_file = os.path.abspath(cfg_file)
if not os.path.isfile(cfg_file):
raise RuntimeError("Could not find config file: {0}".format(cfg_file))
config = config_parser()
config.read(cfg_file)
section = 'suites'
if not config.has_section(section):
raise RuntimeError("ERROR: config file must contain a "
"'{0}' section.".format(section))
suites = {}
for option in config.options(section):
tmp = config.get(section, option)
suites[option] = tmp.split(',')
for s in suites:
suites[s] = [l.strip() for l in suites[s]]
print("Known test suites:")
for s in suites:
print(" {0} : {1}".format(s, ', '.join(suites[s])))
if suite_name not in suites:
raise RuntimeError("ERROR: config file does not contain a test suite '{0}'".format(suite_name))
return suites[suite_name]
def read_externals_description_file(root_dir, file_name):
"""Read a file containing an externals description and
create its internal representation.
"""
root_dir = os.path.abspath(root_dir)
msg = 'In directory : {0}'.format(root_dir)
logging.info(msg)
printlog('Processing externals description file : {0}'.format(file_name))
file_path = os.path.join(root_dir, file_name)
if not os.path.exists(file_name):
if file_name.lower() == "none":
msg = ('INTERNAL ERROR: Attempt to read externals file '
'from {0} when not configured'.format(file_path))
else:
msg = ('ERROR: Model description file, "{0}", does not '
'exist at path:\n {1}\nDid you run from the root of '
'the source tree?'.format(file_name, file_path))
fatal_error(msg)
externals_description = None
if file_name == ExternalsDescription.GIT_SUBMODULES_FILENAME:
externals_description = read_gitmodules_file(root_dir, file_name)
else:
try:
config = config_parser()
config.read(file_path)
externals_description = config
except MissingSectionHeaderError:
# not a cfg file
pass
if externals_description is None:
msg = 'Unknown file format!'
fatal_error(msg)
return externals_description
def setUp(self):
"""Create config object used as basis for all tests
"""
self._config = config_parser()
self.setup_config()
def create_config(self):
"""Create an config object and add the required metadata section
"""
self._config = config_parser()
self.create_metadata()
def loadConfig(dirname):
cp = config_parser()
cp.read(os.path.join(dirname, "test.cfg"))
return cp
def __init__(self, filename):
"""Initialise Variable class.
filename: name of file containing variable definitions."""
dict.__init__(self)
# reading variable configuration file
vars = config_parser()
vars.read(filename)
self.__have_avg = False
for v in vars.sections():
if v == 'VARSET':
for name, value in vars.items(v, raw=True):
module[name] = value
continue
vardef = {}
vardef['name'] = v
for name, value in vars.items(v, raw=True):
vardef[name] = value
if 'type' not in vardef:
vardef['type'] = 'float'
if 'average' in vardef:
if vardef['average'].lower() in ['1', 'true', 't']:
vardef['average'] = True
self.__have_avg = True
else:
vardef['average'] = False
else:
vardef['average'] = False
# handle dims
for d in vardef['dimensions'].split(','):
d = d.strip()
if 'dimlen' in vardef:
dimensions[d] = vardef['dimlen']
if d not in dimensions:
dimensions[d] = '-1'
self.__setitem__(v, vardef)
# handle average
if vardef['average']:
# copy data structure
vardef_avg = vardef.copy()
vardef_avg['average'] = False
vardef_avg['load'] = '0'
vardef_avg['data'] = '%s_%s' % (vardef_avg['data'],
AVERAGE_SUFFIX)
vardef_avg['name'] = '%s_%s' % (vardef_avg['name'],
AVERAGE_SUFFIX)
if 'long_name' in vardef_avg:
vardef_avg['long_name'] = '%s (time average)' % vardef_avg[
'long_name']
if 'cell_methods' in vardef_avg:
vardef_avg[
'cell_methods'] = '%s time: mean over years' % vardef_avg[
'cell_methods']
else:
vardef_avg['cell_methods'] = 'time: mean over years'
vardef_avg['avg_factor'] = 'tavgf'
# and add to dictionary
self.__setitem__('%s_%s' % (v, AVERAGE_SUFFIX), vardef_avg)
def read_gitmodules_file(root_dir, file_name):
# pylint: disable=deprecated-method
# Disabling this check because the method is only used for python2
"""Read a .gitmodules file and convert it to be compatible with an
externals description.
"""
root_dir = os.path.abspath(root_dir)
msg = 'In directory : {0}'.format(root_dir)
logging.info(msg)
printlog('Processing submodules description file : {0}'.format(file_name))
file_path = os.path.join(root_dir, file_name)
if not os.path.exists(file_name):
msg = ('ERROR: submodules description file, "{0}", does not '
'exist at path:\n {1}'.format(file_name, file_path))
fatal_error(msg)
submodules_description = None
externals_description = None
try:
config = config_parser()
if USE_PYTHON2:
config.readfp(LstripReader(file_path), filename=file_name)
else:
config.read_file(LstripReader(file_path), source=file_name)
submodules_description = config
except MissingSectionHeaderError:
# not a cfg file
pass
if submodules_description is None:
msg = 'Unknown file format!'
fatal_error(msg)
else:
# Convert the submodules description to an externals description
externals_description = config_parser()
# We need to grab all the commit hashes for this repo
submods = git_submodule_status(root_dir)
for section in submodules_description.sections():
if section[0:9] == 'submodule':
sec_name = section[9:].strip(' "')
externals_description.add_section(sec_name)
section_items = submodules_description.items(section)
path, url = parse_submodules_desc_section(
section_items, file_path)
if path is None:
msg = 'Submodule {} missing path'.format(sec_name)
fatal_error(msg)
if url is None:
msg = 'Submodule {} missing url'.format(sec_name)
fatal_error(msg)
externals_description.set(sec_name, ExternalsDescription.PATH,
path)
externals_description.set(sec_name,
ExternalsDescription.PROTOCOL, 'git')
externals_description.set(sec_name,
ExternalsDescription.REPO_URL, url)
externals_description.set(sec_name,
ExternalsDescription.REQUIRED,
'True')
git_hash = submods[sec_name]['hash']
externals_description.set(sec_name, ExternalsDescription.HASH,
git_hash)
# Required items
externals_description.add_section(DESCRIPTION_SECTION)
externals_description.set(DESCRIPTION_SECTION, VERSION_ITEM, '1.0.0')
return externals_description
def setUp(self):
"""Create config object used as basis for all tests
"""
self._config = config_parser()
self.setup_config()
