def test_get_with_unknown_section(self):
"""
get() will return `None` for a section name that is not known.
"""
config.Config().cfg.clear()
config.Config()._load_from_file()
self.assertTrue(config.Config().get("Anything") is None)
def teardown_config(self):
os.environ.clear()
os.environ.update(self.orig_env)
# Move the local configuration file back into place if it was stashed
# away.
local_path = "%s/openquake.cfg" % os.path.abspath(os.getcwd())
if os.path.isfile("%s.test_bakk" % local_path):
shutil.move("%s.test_bakk" % local_path, local_path)
config.Config().cfg.clear()
config.Config()._load_from_file()
def test_get_with_known_section(self):
"""
get() will correctly return configuration data for known sections.
"""
content = '''
[E]
f=6
g=h'''
site_path = touch(content=textwrap.dedent(content))
os.environ["OQ_SITE_CFG_PATH"] = site_path
config.Config().cfg.clear()
config.Config()._load_from_file()
self.assertEqual({"f": "6", "g": "h"}, config.Config().get("E"))
def prepare_config(self, section, data=None):
"""Set up a configuration with the given `max_mem` value."""
if data is not None:
data = '\n'.join(["%s=%s" % item for item in data.iteritems()])
content = """
[%s]
%s""" % (section, data)
else:
content = ""
site_path = touch(content=textwrap.dedent(content))
os.environ["OQ_SITE_CFG_PATH"] = site_path
config.Config().cfg.clear()
config.Config()._load_from_file()
def test_get_section_merely_calls_get_on_config_data_dict(self):
"config.get_section() merely makes use of Config().get()" ""
orig_method = config.Config().get
def fake_get(section):
self.assertEqual("f@k3", section)
return {"this": "is", "so": "fake"}
config.Config().get = fake_get
self.assertEqual({
"this": "is",
"so": "fake"
}, config.get_section("f@k3"))
config.Config().get = orig_method
def test_load_from_file_with_global(self):
"""The config data in the global file is loaded correctly."""
content = '''
[A]
a=1
b=c
[B]
b=2'''
site_path = touch(content=textwrap.dedent(content))
os.environ["OQ_SITE_CFG_PATH"] = site_path
config.Config().cfg.clear()
config.Config()._load_from_file()
self.assertEqual(["A", "B"], sorted(config.Config().cfg.keys()))
self.assertEqual({"a": "1", "b": "c"}, config.Config().cfg.get("A"))
self.assertEqual({"b": "2"}, config.Config().cfg.get("B"))
def test_load_from_file_with_local(self):
"""The config data in the local file is loaded correctly."""
content = '''
[C]
c=3
d=e
[D]
d=4'''
local_path = touch(content=textwrap.dedent(content))
os.environ["OQ_LOCAL_CFG_PATH"] = local_path
config.Config().cfg.clear()
config.Config()._load_from_file()
self.assertEqual(["C", "D"], sorted(config.Config().cfg.keys()))
self.assertEqual({"c": "3", "d": "e"}, config.Config().cfg.get("C"))
self.assertEqual({"d": "4"}, config.Config().cfg.get("D"))
def test_get_paths_with_local_env_var_set(self):
"""
_get_paths() will honour the OQ_LOCAL_CFG_PATH
variable.
"""
os.environ["OQ_LOCAL_CFG_PATH"] = "/e/f/g/h"
self.assertEqual(["/etc/openquake/openquake.cfg", "/e/f/g/h"],
config.Config()._get_paths())
def test_is_readable_one_plus_files_have_permissions(self):
"""
When at least one config file is present and we have permission to
read it is_readable() returns `True`.
"""
os.environ["OQ_SITE_CFG_PATH"] = "/etc/passwd"
os.environ["OQ_LOCAL_CFG_PATH"] = "/etc/passwd-"
self.assertTrue(config.Config().is_readable())
def test_is_readable_all_files_lack_permissions(self):
"""
When we miss read permissions for all config files is_readable()
returns `False`.
"""
os.environ["OQ_SITE_CFG_PATH"] = "/etc/sudoers"
os.environ["OQ_LOCAL_CFG_PATH"] = "/etc/passwd-"
self.assertFalse(config.Config().is_readable())
def test_load_from_file_with_local_and_global(self):
"""
The config data in the local and global files is loaded correctly.
"""
content = '''
[A]
a=1
b=c
[B]
b=2'''
site_path = touch(content=textwrap.dedent(content))
os.environ["OQ_SITE_CFG_PATH"] = site_path
content = '''
[C]
c=3
d=e
[D]
d=4'''
local_path = touch(content=textwrap.dedent(content))
os.environ["OQ_LOCAL_CFG_PATH"] = local_path
config.Config().cfg.clear()
config.Config()._load_from_file()
self.assertEqual(["A", "B", "C", "D"],
sorted(config.Config().cfg.keys()))
self.assertEqual({"a": "1", "b": "c"}, config.Config().cfg.get("A"))
self.assertEqual({"b": "2"}, config.Config().cfg.get("B"))
self.assertEqual({"c": "3", "d": "e"}, config.Config().cfg.get("C"))
self.assertEqual({"d": "4"}, config.Config().cfg.get("D"))
def test_get_paths_with_no_environ(self):
"""
_get_paths() will return the hard-coded paths if the OQ_SITE_CFG_PATH
and the OQ_LOCAL_CFG_PATH variables are not set.
"""
self.assertEqual([
"/etc/openquake/openquake.cfg",
"%s/openquake.cfg" % os.path.abspath(os.getcwd())
],
config.Config()._get_paths())
def test_get_paths_with_global_env_var_set(self):
"""
_get_paths() will honour the OQ_SITE_CFG_PATH environment
variable.
"""
os.environ["OQ_SITE_CFG_PATH"] = "/a/b/c/d"
self.assertEqual(
["/a/b/c/d",
"%s/openquake.cfg" % os.path.abspath(os.getcwd())],
config.Config()._get_paths())
def test_load_from_file_with_no_config_files(self):
"""In the absence of config files the `cfg` dict will be empty."""
config.Config().cfg.clear()
config.Config()._load_from_file()
self.assertEqual([], config.Config().cfg.keys())
def run_job(job,
params,
sections,
output_type='db',
log_level='warn',
force_inputs=False,
log_file=None):
"""Given an :class:`openquake.db.models.OqJobProfile` object, create a new
:class:`openquake.db.models.OqJob` object and run the job.
NOTE: The params and sections parameters are temporary but will be required
until we can run calculations purely using Django model objects as
calculator input.
Returns the calculation object when the calculation concludes.
:param job:
:class:`openquake.db.models.OqJob` instance
:param params:
A dictionary of config parameters parsed from the calculation
config file.
:param sections:
A list of sections parsed from the calculation config file.
:param output_type:
'db' or 'xml' (defaults to 'db')
:param str log_level:
One of 'debug', 'info', 'warn', 'error', or 'critical'.
Defaults to 'warn'.
:param bool force_inputs: If `True` the model input files will be parsed
and the resulting content written to the database no matter what.
:param str log_file:
Optional log file location.
:returns:
:class:`openquake.db.models.OqJob` instance.
"""
if not output_type in ('db', 'xml'):
raise RuntimeError("output_type must be 'db' or 'xml'")
job.description = job.profile().description
job.status = 'running'
job.save()
# Clear any counters for this job_id, prior to running the
# job.
# We do this just to make sure all of the counters behave properly and can
# provide accurate data about a calculation in-progress.
stats.delete_job_counters(job.id)
# Make the job/calculation ID generally available.
utils_config.Config().job_id = job.id
serialize_results_to = ['db']
if output_type == 'xml':
serialize_results_to.append('xml')
job_ctxt = JobContext(params,
job.id,
sections=sections,
serialize_results_to=serialize_results_to,
oq_job_profile=job.profile(),
oq_job=job,
log_level=log_level,
force_inputs=force_inputs)
# closing all db connections to make sure they're not shared between
# supervisor and job executor processes. otherwise if one of them closes
# the connection it immediately becomes unavailable for other
close_connection()
job_pid = os.fork()
if not job_pid:
# calculation executor process
try:
logs.init_logs_amqp_send(level=log_level, job_id=job.id)
_launch_job(job_ctxt, sections)
except Exception, ex:
logs.LOG.critical("Calculation failed with exception: '%s'" %
str(ex))
job.status = 'failed'
job.save()
raise
else:
job.status = 'succeeded'
job.save()
return
def tearDown(self):
config.Config().cfg.clear()
config.Config()._load_from_file()
def test_is_readable_no_file_present(self):
"""When no config file is present is_readable() returns `False`."""
os.environ["OQ_SITE_CFG_PATH"] = "/this/does/not/exist.cfg"
os.environ["OQ_LOCAL_CFG_PATH"] = "/nor/does/this.cfg"
self.assertFalse(config.Config().is_readable())
def _maintain_debug_stats(self):
"""Capture the file written if debug statistics are turned on."""
key = stats.key_name(config.Config().job_id,
*stats.STATS_KEYS["hcls_xmlcurvewrites"])
if key:
stats.kvs_op("rpush", key, self.path)