def test_add_unknown_keyowrds(self):
parser = ConfigParser( )
with TestAreaContext("config/parse4"):
with open("config","w") as fileH:
fileH.write("SETTINGS A 100.1\n")
fileH.write("SETTINGS B 200 STRING1 STRING2\n")
fileH.write("SETTINGS C 300\n")
fileH.write("SETTINGS D False\n")
content = parser.parse("config", unrecognized=UnrecognizedEnum.CONFIG_UNRECOGNIZED_ADD)
self.assertIn("SETTINGS", content)
item = content["SETTINGS"]
self.assertEqual(len(item), 4)
nodeA = item[0]
self.assertEqual(nodeA[0], "A")
self.assertEqual(nodeA[1], "100.1")
self.assertEqual(len(nodeA), 2)
nodeB = item[1]
self.assertEqual(nodeB[0], "B")
self.assertEqual(nodeB[1], "200")
self.assertEqual(nodeB[3], "STRING2")
self.assertEqual(len(nodeB), 4)
self.assertEqual(len(content), 4)
def test_parse(self):
conf = ConfigParser()
conf.add("FIELD", False)
schema_item = conf.add("RSH_HOST", False)
self.assertIsInstance(schema_item, SchemaItem)
test_path = self.createTestPath("local/config/simple_config")
content = conf.parse(test_path, unrecognized=UnrecognizedEnum.CONFIG_UNRECOGNIZED_IGNORE)
self.assertTrue(content.isValid())
content_item = content["RSH_HOST"]
self.assertIsInstance(content_item, ContentItem)
self.assertEqual(len(content_item), 1)
with self.assertRaises(TypeError):
content_item["BJARNE"]
with self.assertRaises(IndexError):
content_item[10]
content_node = content_item[0]
self.assertIsInstance(content_node, ContentNode)
self.assertEqual(len(content_node), 2)
self.assertEqual(content_node[1], "be-lx633214:2")
self.assertEqual(content_node.content(sep=","), "be-lx655082:2,be-lx633214:2")
self.assertEqual(content_node.content(), "be-lx655082:2 be-lx633214:2")
content_item = content["FIELD"]
self.assertEqual(len(content_item), 5)
with self.assertRaises(IOError):
conf.parse("DoesNotExits")
def test_parse_deprecated(self):
conf = ConfigParser()
item = conf.add("INT", value_type=ContentTypeEnum.CONFIG_INT)
msg = "ITEM INT IS DEPRECATED"
item.setDeprecated(msg)
with TestAreaContext("config/parse2"):
with open("config","w") as fileH:
fileH.write("INT 100\n")
content = conf.parse("config" )
self.assertTrue(content.isValid())
warnings = content.getWarnings()
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0], msg)
def _alloc_from_content(self, user_config_file=None, config=None):
if user_config_file is not None:
# initialize configcontent if user_file provided
parser = ConfigParser()
config_content = self._alloc_config_content(user_config_file, parser)
config_dir = config_content.getValue(ConfigKeys.CONFIG_DIRECTORY)
else:
config_dir = os.getcwd()
config_content = self._build_config_content(config)
if self.errors:
raise ValueError("Error loading configuration: " + str(self._errors))
subst_config = SubstConfig(config_content=config_content)
site_config = SiteConfig(config_content=config_content)
rng_config = RNGConfig(config_content=config_content)
analysis_config = AnalysisConfig(config_content=config_content)
ecl_config = EclConfig(config_content=config_content)
queue_config = QueueConfig(config_content=config_content)
ert_workflow_list = ErtWorkflowList(
subst_list=subst_config.subst_list, config_content=config_content
)
hook_manager = HookManager(
workflow_list=ert_workflow_list, config_content=config_content
)
ert_templates = ErtTemplates(
parent_subst=subst_config.subst_list, config_content=config_content
)
ensemble_config = EnsembleConfig(
config_content=config_content,
grid=ecl_config.getGrid(),
refcase=ecl_config.getRefcase(),
)
model_config = ModelConfig(
data_root=config_dir,
joblist=site_config.get_installed_jobs(),
last_history_restart=ecl_config.getLastHistoryRestart(),
refcase=ecl_config.getRefcase(),
config_content=config_content,
)
return [
subst_config,
site_config,
rng_config,
analysis_config,
ert_workflow_list,
hook_manager,
ert_templates,
ecl_config,
ensemble_config,
model_config,
queue_config,
], config_dir
def test_item_types(self):
with TestAreaContext("config/types") as test_area:
with open("config", "w") as f:
f.write("TYPE_ITEM 10 3.14 TruE String file\n")
conf = ConfigParser()
self.assertEqual(0, len(conf))
schema_item = conf.add("TYPE_ITEM", False)
schema_item.iset_type(0, ContentTypeEnum.CONFIG_INT)
schema_item.iset_type(1, ContentTypeEnum.CONFIG_FLOAT)
schema_item.iset_type(2, ContentTypeEnum.CONFIG_BOOL)
schema_item.iset_type(3, ContentTypeEnum.CONFIG_STRING)
schema_item.iset_type(4, ContentTypeEnum.CONFIG_PATH)
self.assertEqual(1, len(conf))
self.assertNotIn("TYPE_XX", conf)
self.assertIn("TYPE_ITEM", conf)
content = conf.parse("config")
type_item = content["TYPE_ITEM"][0]
int_value = type_item[0]
self.assertEqual(int_value, 10)
self.assertEqual(type_item.igetString(0), "10")
float_value = type_item[1]
self.assertEqual(float_value, 3.14)
self.assertEqual(type_item.igetString(1), "3.14")
bool_value = type_item[2]
self.assertEqual(bool_value, True)
self.assertEqual(type_item.igetString(2), "TruE")
string_value = type_item[3]
self.assertEqual(string_value, "String")
self.assertEqual(type_item.igetString(3), "String")
path_value = type_item[4]
self.assertEqual(path_value, "file")
self.assertEqual(type_item.igetString(4), "file")
# test __getitem__
self.assertTrue(conf['TYPE_ITEM'])
with self.assertRaises(KeyError):
_ = conf['TYPE_XX']
self.assertIn('ConfigParser', repr(conf))
self.assertIn('size=1', repr(conf))
def test_parse_dotdot_relative(self):
conf = ConfigParser()
schema_item = conf.add("EXECUTABLE", False)
schema_item.iset_type(0, ContentTypeEnum.CONFIG_PATH)
with TestAreaContext("config/parse_dotdot"):
os.makedirs("cwd/jobs")
os.makedirs("eclipse/bin")
script_path = os.path.join(os.getcwd(), "eclipse/bin/script.sh")
with open(script_path, "w") as f:
f.write("This is a test script")
with open("cwd/jobs/JOB", "w") as fileH:
fileH.write("EXECUTABLE ../../eclipse/bin/script.sh\n")
os.makedirs("cwd/ert")
os.chdir("cwd/ert")
content = conf.parse("../jobs/JOB")
item = content["EXECUTABLE"]
node = item[0]
self.assertEqual(script_path, node.getPath())
def _build_config_content(self, config):
self._failed_keys = {}
defines, config_dir, config_list = self._extract_config(config)
config_parser = ConfigParser()
ResConfig.init_config_parser(config_parser)
config_content = ConfigContent(None)
config_content.setParser(config_parser)
# Insert defines
for key in defines:
config_content.add_define(key, defines[key])
# Insert key values
if not os.path.exists( config_dir ):
raise IOError("The configuration direcetory: %s does not exist" % config_dir)
path_elm = config_content.create_path_elm(config_dir)
add_key_value = lambda key, value : config_parser.add_key_value(
config_content,
key,
StringList([key] + value),
path_elm=path_elm)
for key, value in config_list:
if isinstance(value, str):
value = [value]
if not isinstance(value, list):
raise ValueError("Expected value to be str or list, was %r" % (type(value)))
ok = add_key_value(key, value)
if not ok:
self._failed_keys[key] = value
config_parser.validate(config_content)
self._errors = list(config_content.getErrors())
return config_content
def _build_config_content(self, config):
self._failed_keys = {}
defines, config_dir, config_list = self._extract_config(config)
config_parser = ConfigParser()
ResConfig.init_config_parser(config_parser)
config_content = ConfigContent(None)
config_content.setParser(config_parser)
if config_dir is None:
raise ValueError("Expected config to specify %s" %
ConfigKeys.CONFIG_DIRECTORY)
# Insert defines
for key in defines:
config_content.add_define(key, defines[key])
# Insert key values
path_elm = config_content.create_path_elm(config_dir)
add_key_value = lambda key, value: config_parser.add_key_value(
config_content, key, StringList([key] + value), path_elm=path_elm)
for key, value in config_list:
if isinstance(value, str):
value = [value]
if not isinstance(value, list):
raise ValueError("Expected value to be str or list, was %r" %
(type(value)))
ok = add_key_value(key, value)
if not ok:
self._failed_keys[key] = value
config_parser.validate(config_content)
self._errors = list(config_content.getErrors())
return config_content
def test_parse_invalid(self):
conf = ConfigParser()
conf.add("INT", value_type=ContentTypeEnum.CONFIG_INT)
with TestAreaContext("config/parse2"):
with open("config","w") as fileH:
fileH.write("INT xx\n")
with self.assertRaises(ValueError):
conf.parse("config")
content = conf.parse("config", validate=False)
self.assertFalse(content.isValid())
self.assertEqual(len(content.getErrors()), 1)
def test_parse(self):
config_file = """
RSH_HOST some-hostname:2 other-hostname:2
FIELD PRESSURE DYNAMIC
FIELD SWAT DYNAMIC MIN:0 MAX:1
FIELD SGAS DYNAMIC MIN:0 MAX:1
FIELD RS DYNAMIC MIN:0
FIELD RV DYNAMIC MIN:0.0034"""
with open("simple_config", "w") as fout:
fout.write(config_file)
conf = ConfigParser()
conf.add("FIELD", False)
schema_item = conf.add("RSH_HOST", False)
self.assertIsInstance(schema_item, SchemaItem)
content = conf.parse(
"simple_config",
unrecognized=UnrecognizedEnum.CONFIG_UNRECOGNIZED_IGNORE)
self.assertTrue(content.isValid())
content_item = content["RSH_HOST"]
self.assertIsInstance(content_item, ContentItem)
self.assertEqual(len(content_item), 1)
with self.assertRaises(TypeError):
content_item["BJARNE"]
with self.assertRaises(IndexError):
content_item[10]
content_node = content_item[0]
self.assertIsInstance(content_node, ContentNode)
self.assertEqual(len(content_node), 2)
self.assertEqual(content_node[1], "other-hostname:2")
self.assertEqual(content_node.content(sep=","),
"some-hostname:2,other-hostname:2")
self.assertEqual(content_node.content(),
"some-hostname:2 other-hostname:2")
content_item = content["FIELD"]
self.assertEqual(len(content_item), 5)
with self.assertRaises(IOError):
conf.parse("DoesNotExits")
def config_parser(cls):
parser = ConfigParser()
cls._init_parser(parser)
return parser
def test_settings(self):
cs = ConfigSettings("SETTINGS")
with self.assertRaises(TypeError):
cs.addSetting("SETTING1", ContentTypeEnum.CONFIG_INT, 100.67)
self.assertFalse("SETTING1" in cs)
cs.addSetting("SETTING2", ContentTypeEnum.CONFIG_INT, 100)
self.assertTrue("SETTING2" in cs)
with self.assertRaises(KeyError):
value = cs["NO_SUCH_KEY"]
self.assertEqual(cs["SETTING2"], 100)
with self.assertRaises(KeyError):
cs["NO_SUCH_KEY"] = 100
parser = ConfigParser()
cs = ConfigSettings("SETTINGS")
cs.addSetting("A", ContentTypeEnum.CONFIG_INT, 1)
cs.addSetting("B", ContentTypeEnum.CONFIG_INT, 1)
cs.addSetting("C", ContentTypeEnum.CONFIG_INT, 1)
cs.initParser(parser)
with TestAreaContext("config/parse3"):
with open("config","w") as fileH:
fileH.write("SETTINGS A 100\n")
fileH.write("SETTINGS B 200\n")
fileH.write("SETTINGS C 300\n")
content = parser.parse("config")
cs.apply(content)
self.assertEqual(cs["A"], 100)
self.assertEqual(cs["B"], 200)
self.assertEqual(cs["C"], 300)
keys = cs.keys()
self.assertTrue("A" in keys)
self.assertTrue("B" in keys)
self.assertTrue("C" in keys)
self.assertEqual(len(keys), 3)
cs = ConfigSettings("SETTINGS")
cs.addDoubleSetting("A", 1.0)
self.assertEqual(ContentTypeEnum.CONFIG_FLOAT, cs.getType("A"))
cs = ConfigSettings("SETTINGS")
cs.addDoubleSetting("A", 1.0)
cs.addIntSetting("B", 1)
cs.addStringSetting("C", "1")
cs.addBoolSetting("D", True)
cs.initParser(parser)
with TestAreaContext("config/parse4"):
with open("config","w") as fileH:
fileH.write("SETTINGS A 100.1\n")
fileH.write("SETTINGS B 200\n")
fileH.write("SETTINGS C 300\n")
fileH.write("SETTINGS D False\n")
content = parser.parse("config")
cs.apply(content)
self.assertEqual(cs["A"], 100.1)
self.assertEqual(cs["B"], 200)
self.assertEqual(cs["C"], "300")
self.assertEqual(cs["D"], False)
with self.assertRaises(Exception):
cs["A"] = "Hei"
def test_parser_content(self):
conf = ConfigParser()
conf.add("KEY2", False)
schema_item = conf.add("KEY", False)
schema_item.iset_type(2, ContentTypeEnum.CONFIG_INT)
schema_item.iset_type(3, ContentTypeEnum.CONFIG_BOOL)
schema_item.iset_type(4, ContentTypeEnum.CONFIG_FLOAT)
schema_item.iset_type(5, ContentTypeEnum.CONFIG_PATH)
schema_item = conf.add("NOT_IN_CONTENT", False)
with TestAreaContext("config/parse2"):
with open("config","w") as fileH:
fileH.write("KEY VALUE1 VALUE2 100 True 3.14 path/file.txt\n")
cwd0 = os.getcwd()
os.makedirs("tmp")
os.chdir("tmp")
content = conf.parse("../config")
d = content.as_dict()
self.assertTrue(content.isValid())
self.assertTrue("KEY" in content)
self.assertFalse("NOKEY" in content)
self.assertEqual( cwd0, content.get_config_path( ))
keys = content.keys()
self.assertEqual(len(keys), 1)
self.assertIn("KEY", keys)
d = content.as_dict()
self.assertIn("KEY", d)
item_list = d["KEY"]
self.assertEqual(len(item_list), 1)
l = item_list[0]
self.assertEqual(l[0], "VALUE1")
self.assertEqual(l[1], "VALUE2")
self.assertEqual(l[2], 100)
self.assertEqual(l[3], True)
self.assertEqual(l[4], 3.14)
self.assertEqual(l[5], "../path/file.txt")
self.assertFalse("NOT_IN_CONTENT" in content)
item = content["NOT_IN_CONTENT"]
self.assertEqual(len(item), 0)
with self.assertRaises(KeyError):
content["Nokey"]
item = content["KEY"]
self.assertEqual(len(item), 1)
line = item[0]
with self.assertRaises(TypeError):
line.getPath(4)
with self.assertRaises(TypeError):
line.getPath()
rel_path = line.getPath(index=5, absolute=False)
self.assertEqual(rel_path, "../path/file.txt")
get = line[5]
self.assertEqual(get, "../path/file.txt")
abs_path = line.getPath(index=5)
self.assertEqual(abs_path, os.path.join(cwd0, "path/file.txt"))
rel_path = line.getPath(index=5, absolute=False, relative_start="../")
self.assertEqual(rel_path, "path/file.txt")
with self.assertRaises(IndexError):
item[10]
node = item[0]
self.assertEqual(len(node), 6)
with self.assertRaises(IndexError):
node[6]
self.assertEqual(node[0], "VALUE1")
self.assertEqual(node[1], "VALUE2")
self.assertEqual(node[2], 100)
self.assertEqual(node[3], True)
self.assertEqual(node[4], 3.14)
self.assertEqual(content.getValue("KEY", 0, 1), "VALUE2")
self.assertEqual(_iget(content, "KEY", 0, 1), "VALUE2")
self.assertEqual(content.getValue("KEY", 0, 2), 100)
self.assertEqual(_iget_as_int(content, "KEY", 0, 2), 100)
self.assertEqual(content.getValue("KEY", 0, 3), True)
self.assertEqual(_iget_as_bool(content, "KEY", 0, 3), True)
self.assertEqual(content.getValue("KEY", 0, 4), 3.14)
self.assertEqual(_iget_as_double(content, "KEY", 0, 4), 3.14)
self.assertIsNone(_safe_iget(content, "KEY2", 0, 0))
self.assertEqual( _get_occurences(content, "KEY2"), 0)
self.assertEqual( _get_occurences(content, "KEY"), 1)
self.assertEqual( _get_occurences(content, "MISSING-KEY"), 0)
def __init__(self,
user_config_file=None,
config=None,
throw_on_error=True):
self._errors, self._failed_keys = None, None
self._assert_input(user_config_file, config, throw_on_error)
if config is not None:
config_content = self._build_config_content(config)
elif user_config_file is not None:
parser = ConfigParser()
config_content = self._alloc_config_content(
user_config_file, parser)
else:
raise ValueError("No config provided")
if self.errors and throw_on_error:
raise ValueError("Error loading configuration: " +
str(self._errors))
config_dir = config_content.getValue(ConfigKeys.CONFIG_DIRECTORY)
subst_config = SubstConfig(config_content=config_content)
site_config = SiteConfig(config_content=config_content)
rng_config = RNGConfig(config_content=config_content)
analysis_config = AnalysisConfig(config_content=config_content)
ecl_config = EclConfig(config_content=config_content)
log_config = LogConfig(config_content=config_content)
queue_config = QueueConfig(config_content=config_content)
ert_workflow_list = ErtWorkflowList(
ert_workflow_list=subst_config.subst_list,
config_content=config_content)
hook_manager = HookManager(workflow_list=ert_workflow_list,
config_content=config_content)
ert_templates = ErtTemplates(parent_subst=subst_config.subst_list,
config_content=config_content)
ensemble_config = EnsembleConfig(config_content=config_content,
grid=ecl_config.getGrid(),
refcase=ecl_config.getRefcase())
model_config = ModelConfig(
config_content=config_content,
data_root=config_dir,
joblist=site_config.get_installed_jobs(),
last_history_restart=ecl_config.getLastHistoryRestart(),
sched_file=ecl_config._get_sched_file(),
refcase=ecl_config.getRefcase())
configs = [
subst_config, site_config, rng_config, analysis_config,
ert_workflow_list, hook_manager, ert_templates, ecl_config,
ensemble_config, model_config, log_config, queue_config
]
c_ptr = None
for conf in configs:
conf.convertToCReference(None)
c_ptr = self._alloc_full(config_dir, user_config_file, *configs)
if c_ptr:
super(ResConfig, self).__init__(c_ptr)
else:
raise ValueError(
'Failed to construct ResConfig instance from %r.' %
(user_config_file if user_config_file else config))
def parse(fname):
"""Takes a configuration object containing grid file path, restart file
path and a list of wellpath-files and their restart dates. Returns a
SnapWell object, containing this info.
"""
conf = ConfigParser()
SnapConfig.addConfigItemPath(conf, 'GRID')
SnapConfig.addConfigItemPath(conf, 'RESTART')
SnapConfig.addConfigItemPath(conf, 'INIT')
SnapConfig.addConfigItemPath(conf, 'OUTPUT')
overwrite_item = conf.add("OVERWRITE",
False) # the last OVERWRITE specified counts
overwrite_item.iset_type(0, ContentTypeEnum.CONFIG_STRING)
SnapConfig.addConfigItemFloat(conf, 'OWC_OFFSET')
SnapConfig.addConfigItemFloat(conf, 'DELTA_Z')
owc_def_item = conf.add(
'OWC_DEFINITION') # e.g. OWC_DEFINITION SWAT 0.7
owc_def_item.iset_type(0, ContentTypeEnum.CONFIG_STRING)
owc_def_item.iset_type(1, ContentTypeEnum.CONFIG_FLOAT)
log_item = conf.add('LOG')
log_item.iset_type(0, ContentTypeEnum.CONFIG_STRING)
wellpath_item = conf.add("WELLPATH",
True) # a series of WELLPATH/DATE pairs
wellpath_item.iset_type(0, ContentTypeEnum.CONFIG_PATH)
wellpath_item.iset_type(1, ContentTypeEnum.CONFIG_STRING)
content = conf.parse(fname)
# Grid
gridFile = tryGetPath(content, 'GRID', 0, 0)
if not gridFile:
logging.info('No GRID file specified?')
raise ValueError(
'Could not load GRID file from Snapwell config file.')
# Restart
restartFile = tryGetPath(content, 'RESTART', 0, 0)
if not restartFile:
logging.info('No RESTART file specified?')
raise ValueError(
'Could not load RESTART file from Snapwell config file.')
# init
initFile = None
if "INIT" in content:
initFile = tryGetPath(content, 'INIT', 0, 0)
s = SnapConfig(gridFile, restartFile, initFile)
# output
outputPath = None
if "OUTPUT" in content:
outputPath = tryGetPath(content, 'OUTPUT', 0, 0)
if path.exists(outputPath):
if path.isfile(outputPath):
raise ValueError(
'Provided output folder is a file. Either delete file, or set a different output folder: %s'
% outputPath)
else:
s.setOutput(outputPath)
else:
makedirs(outputPath)
s.setOutput(outputPath)
# overwrite
overwrite = False
if 'OVERWRITE' in content:
overwrite = tryGet(content, 'OVERWRITE', 0, 0)
try:
if overwrite.strip().lower() == 'true':
overwrite = True
except Exception as err:
logging.warning('Ill specified overwrite flag: "%s".' % err)
s.setOverwrite(overwrite)
SnapConfig.tryset(s.setDeltaZ, 'DELTA_Z', content)
SnapConfig.tryset(s.setOwcOffset, 'OWC_OFFSET', content)
# Loading OWC_DEFINITION
if 'OWC_DEFINITION' in content:
try:
if len(content['OWC_DEFINITION'][0]) != 2:
logging.warning(
'Wrong number of arguments in OWC_DEFINITION. Needs to be e.g. SWAT 0.7, got %s'
% str(content['OWC_DEFINITION'][0]))
else:
owc_kw = tryGet(content, 'OWC_DEFINITION', 0, 0)
owc_val = tryGetFloat(content, 'OWC_DEFINITION', 0, 1)
s.setOwcDefinition((owc_kw, owc_val))
except Exception as err:
logging.warning('Ill specified OWC_DEFINITION keyword: "%s".' %
err)
else:
logging.info('Using OWC definition %s' % str(s.owcDefinition()))
if 'LOG' in content:
for i in range(len(content['LOG'])):
line = content['LOG'][i]
num_tokens = len(line)
if num_tokens < 1:
raise ValueError('Missing data in LOG %d: "%s".' %
(i + 1, str(line)))
s.addLogKeyword(tryGet(content, 'LOG', i, 0))
# Loading the wellpath file names
if 'WELLPATH' not in content:
logging.warning('No wellpaths provided in Snapwell config file.')
return s
for i in range(len(content['WELLPATH'])):
wp_line = content['WELLPATH'][i]
num_tokens = len(wp_line)
if num_tokens < 2:
raise ValueError('Missing data in WELLPATH %d: "%s".' %
(i + 1, str(wp_line)))
fname = tryGetPath(content, 'WELLPATH', i, 0)
date_str = tryGet(content, 'WELLPATH', i, 1)
if num_tokens > 2 and num_tokens != 4:
raise ValueError(
'WELLPATH format error. Need 2 or 4 tokens, got %d tokens: %s'
% (num_tokens, str(wp_line)))
depth_type = None
depth = -1
if num_tokens == 4:
depth_type = tryGet(content, 'WELLPATH', i, 2)
depth = tryGetFloat(content, 'WELLPATH', i, 3)
date = None
try:
date = parse_date(date_str)
except ValueError as err:
logging.info(str(err))
if date is None:
raise ValueError(
'Could not read date from wellpath %d. Got date string "%s".'
% (i + 1, str(date_str)))
if depth_type:
s.append((fname, date, depth_type, depth))
else:
s.append((fname, date))
return s
