def setUp(self):
self.mixin = self.create_job_with_mixin({'CALCULATION_MODE': 'Hazard'},
opensha.ClassicalMixin)
# Store the canned result data in the KVS.
key = self.mixin.job_id
for realization in xrange(2):
key = "%s/%s" % (self.mixin.job_id, realization + 1)
TestStore.put(key, self.mock_results[realization])
self.keys.append(key)
LOG.debug("keys = '%s'" % self.keys)
# Initialize the mixin instance.
self.mixin.params = dict(NUMBER_OF_LOGIC_TREE_SAMPLES=2,
WIDTH_OF_MFD_BIN=1)
self.mixin.calc = self.FakeLogicTreeProcessor()
self.mixin.cache = dict()
def test_map_serializer_called_when_configured(self):
"""
The mean map serialization function is called when the POES
parameter is specified in the configuration file.
"""
def fake_serializer(*args):
"""Fake serialization function to be used in this test."""
# Check that the data returned is the one we expect for the current
# realization.
fake_serializer.number_of_calls += 1
def fake_map_func(*args):
return list(xrange(3))
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.job_ctxt.params["POES"] = "0.6 0.8"
self.calculator.do_means(self.sites,
1,
curve_serializer=lambda _: True,
curve_task=test_data_reflector,
map_func=fake_map_func,
map_serializer=fake_serializer)
self.assertEqual(1, fake_serializer.number_of_calls)
def test_distribute_with_ignore_result_set(self):
"""
The specified number of subtasks is actually spawned even for tasks
with ignore_result=True and these run and complete.
Since the results of the tasks are ignored, the only way to know that
they ran and completed is to verify that the data they were supposed
to write the key value store is actually there.
"""
def value(key):
"""Construct a test value for the given key."""
return key[-3:] * 2
keys = ["irtc:%s" % str(uuid.uuid4())[:8] for _ in xrange(5)]
values = [value(uid) for uid in keys]
data = zip(keys, values)
result = tasks.distribute(ignore_result, ("data", [[d] for d in data]))
# An empty list is returned for tasks with ignore_result=True
# and no asynchronous task handler function.
self.assertEqual(False, bool(result))
# Give the tasks a bit of time to complete.
time.sleep(0.1)
for key, value in data:
self.assertEqual(value, TestStore.get(key))
def test_map_serializer_called_when_configured(self):
"""
The mean map serialization function is called when the POES
parameter is specified in the configuration file.
"""
def fake_serializer(*args):
"""Fake serialization function to be used in this test."""
# Check that the data returned is the one we expect for the current
# realization.
fake_serializer.number_of_calls += 1
def fake_map_func(*args):
return list(xrange(3))
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.job_ctxt.params["POES"] = "0.6 0.8"
self.calculator.do_means(self.sites, 1,
curve_serializer=lambda _: True,
curve_task=test_data_reflector,
map_func=fake_map_func,
map_serializer=fake_serializer)
self.assertEqual(1, fake_serializer.number_of_calls)
def test_distribute_with_ignore_result_set(self):
"""
The specified number of subtasks is actually spawned even for tasks
with ignore_result=True and these run and complete.
Since the results of the tasks are ignored, the only way to know that
they ran and completed is to verify that the data they were supposed
to write the key value store is actually there.
"""
def value(key):
"""Construct a test value for the given key."""
return key[-3:] * 2
keys = ["irtc:%s" % str(uuid.uuid4())[:8] for _ in xrange(5)]
values = [value(uid) for uid in keys]
data = zip(keys, values)
result = tasks.distribute(ignore_result, ("data", [[d] for d in data]))
# An empty list is returned for tasks with ignore_result=True
# and no asynchronous task handler function.
self.assertEqual(False, bool(result))
# Give the tasks a bit of time to complete.
time.sleep(0.1)
for key, value in data:
self.assertEqual(value, TestStore.get(key))
def test_map_serializer_called_when_configured(self):
"""
The quantile map serialization function is called when the
POES parameter is specified in the configuration file.
"""
def fake_serializer(*args):
"""Fake serialization function to be used in this test."""
fake_serializer.number_of_calls += 1
fake_serializer.number_of_calls = 0
def fake_map_func(*args):
return ["quantile_hazard_map!10!-122.9!38.0!0.2",
"quantile_hazard_map!10!-121.9!38.0!0.2",
"quantile_hazard_map!10!-122.8!38.0!0.4",
"quantile_hazard_map!10!-121.8!38.0!0.4"]
key = TestStore.put(self.job_ctxt.job_id,
self.mock_results)
self.keys.append(key)
self.job_ctxt.params["POES"] = "0.6 0.8"
self.calculator.do_quantiles(
self.sites, 1, [0.2, 0.4],
curve_serializer=lambda _, __: True,
curve_task=test_data_reflector,
map_func=fake_map_func,
map_serializer=fake_serializer)
# The serializer is called once for each quantile.
self.assertEqual(2, fake_serializer.number_of_calls)
def test_map_serializer_called_when_configured(self):
"""
The quantile map serialization function is called when the
POES parameter is specified in the configuration file.
"""
def fake_serializer(*args):
"""Fake serialization function to be used in this test."""
fake_serializer.number_of_calls += 1
fake_serializer.number_of_calls = 0
def fake_map_func(*args):
return [
"quantile_hazard_map!10!-122.9!38.0!0.2",
"quantile_hazard_map!10!-121.9!38.0!0.2",
"quantile_hazard_map!10!-122.8!38.0!0.4",
"quantile_hazard_map!10!-121.8!38.0!0.4"
]
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.job_ctxt.params["POES"] = "0.6 0.8"
self.calculator.do_quantiles(self.sites,
1, [0.2, 0.4],
curve_serializer=lambda _, __: True,
curve_task=test_data_reflector,
map_func=fake_map_func,
map_serializer=fake_serializer)
# The serializer is called once for each quantile.
self.assertEqual(2, fake_serializer.number_of_calls)
def ath(data):
"""
An asynchronous task handler function that converts all task
results to upper case and returns the list of keys found.
"""
items_expected = len(data)
items_found = []
while len(items_found) < items_expected:
for key, _ in data:
if key in items_found:
continue
value = TestStore.get(key)
if value is not None:
TestStore.set(key, value.upper())
items_found.append(key)
time.sleep(0.05)
return items_found
def ath(data):
"""
An asynchronous task handler function that converts all task
results to upper case and returns the list of keys found.
"""
items_expected = len(data)
items_found = []
while len(items_found) < items_expected:
for key, _ in data:
if key in items_found:
continue
value = TestStore.get(key)
if value is not None:
TestStore.set(key, value.upper())
items_found.append(key)
time.sleep(0.05)
return items_found
def setUp(self):
params = dict(
CALCULATION_MODE='Hazard',
SOURCE_MODEL_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_SRC_MODEL_LT,
GMPE_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_GMPE_LT,
BASE_PATH=SIMPLE_FAULT_BASE_PATH, OUTPUT_DIR="output",
NUMBER_OF_LOGIC_TREE_SAMPLES=2, WIDTH_OF_MFD_BIN=1)
self.job_ctxt = create_job(params, job_id=99)
self.calculator = classical.ClassicalHazardCalculator(self.job_ctxt)
# Store the canned result data in the KVS.
key = self.job_ctxt.job_id
for realization in xrange(2):
key = "%s/%s" % (self.job_ctxt.job_id, realization + 1)
TestStore.put(key, self.mock_results[realization])
self.keys.append(key)
LOG.debug("keys = '%s'" % self.keys)
self.calculator.calc = self.FakeLogicTreeProcessor()
self.calculator.cache = dict()
def setUp(self):
params = dict(
CALCULATION_MODE='Hazard',
SOURCE_MODEL_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_SRC_MODEL_LT,
GMPE_LOGIC_TREE_FILE_PATH=SIMPLE_FAULT_GMPE_LT,
BASE_PATH=SIMPLE_FAULT_BASE_PATH, OUTPUT_DIR="output",
NUMBER_OF_LOGIC_TREE_SAMPLES=2, WIDTH_OF_MFD_BIN=1)
self.job_ctxt = create_job(params)
self.calculator = classical.ClassicalHazardCalculator(self.job_ctxt)
# Store the canned result data in the KVS.
key = self.job_ctxt.job_id
for realization in xrange(2):
key = "%s/%s" % (self.job_ctxt.job_id, realization + 1)
TestStore.put(key, self.mock_results[realization])
self.keys.append(key)
LOG.debug("keys = '%s'" % self.keys)
self.calculator.calc = self.FakeLogicTreeProcessor()
self.calculator.cache = dict()
def test_curve_serializer_called_when_passed(self):
"""The passed quantile curve serialization function is called."""
def fake_serializer(**kwargs):
"""Fake serialization function to be used in this test."""
fake_serializer.number_of_calls += 1
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.calculator.do_quantiles(
self.sites, 1, [0.2, 0.4], curve_serializer=fake_serializer,
curve_task=test_async_data_reflector)
# The serializer is called only once (for all quantiles).
self.assertEqual(1, fake_serializer.number_of_calls)
def test_curve_serializer_called_when_passed(self):
"""The passed quantile curve serialization function is called."""
def fake_serializer(**kwargs):
"""Fake serialization function to be used in this test."""
fake_serializer.number_of_calls += 1
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.calculator.do_quantiles(self.sites,
1, [0.2, 0.4],
curve_serializer=fake_serializer,
curve_task=test_async_data_reflector)
# The serializer is called only once (for all quantiles).
self.assertEqual(1, fake_serializer.number_of_calls)
def test_missing_map_serializer_assertion(self):
"""
When the quantile map serialization function is not set an
`AssertionError` is raised.
TODO: once everyone is on python rev. > 2.7 extend the test to check
for the specific assertion message.
"""
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.job_ctxt.params["POES"] = "0.6 0.8"
self.assertRaises(
AssertionError, self.calculator.do_quantiles,
self.sites, 1, [0.5],
curve_serializer=lambda _, __: True,
curve_task=test_data_reflector, map_func=lambda _: [1, 2, 3])
def test_missing_map_serializer_assertion(self):
"""
When the quantile map serialization function is not set an
`AssertionError` is raised.
TODO: once everyone is on python rev. > 2.7 extend the test to check
for the specific assertion message.
"""
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.job_ctxt.params["POES"] = "0.6 0.8"
self.assertRaises(
AssertionError, self.calculator.do_quantiles,
self.sites, 1, [0.5],
curve_serializer=lambda _, __: True,
curve_task=test_data_reflector, map_func=lambda _: [1, 2, 3])
def test_missing_map_function_assertion(self):
"""
When the mean map calculation function is not set an `AssertionError`
is raised.
TODO: once everyone is on python rev. > 2.7 extend the test to check
for the specific assertion message.
"""
key = TestStore.put(self.mixin.job_id, self.mock_results)
self.keys.append(key)
self.mixin.params["POES"] = "0.6 0.8"
self.assertRaises(
AssertionError, self.mixin.do_means,
self.sites, 1,
curve_serializer=lambda _: True, curve_task=test_data_reflector,
map_serializer=lambda _: True, map_func=None)
def test_curve_serializer_called_when_passed(self):
"""The passed mean curve serialization function is called."""
def fake_serializer(sites, **kwargs):
"""Fake serialization function to be used in this test."""
self.assertEqual(self.sites, sites)
fake_serializer.number_of_calls += 1
# Count the number of invocations using this property of the fake
# serializer function.
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.calculator.do_means(
self.sites, 1, curve_serializer=fake_serializer,
curve_task=test_async_data_reflector)
self.assertEqual(1, fake_serializer.number_of_calls)
def test_curve_serializer_called_when_passed(self):
"""The passed mean curve serialization function is called."""
def fake_serializer(sites, **kwargs):
"""Fake serialization function to be used in this test."""
self.assertEqual(self.sites, sites)
fake_serializer.number_of_calls += 1
# Count the number of invocations using this property of the fake
# serializer function.
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.calculator.do_means(self.sites,
1,
curve_serializer=fake_serializer,
curve_task=test_async_data_reflector)
self.assertEqual(1, fake_serializer.number_of_calls)
def test_map_serializer_not_called_unless_configured(self):
"""
The mean map serialization function is not called unless the
POES parameter was specified in the configuration file.
"""
def fake_serializer(kvs_keys):
"""Fake serialization function to be used in this test."""
# Check that the data returned is the one we expect for the current
# realization.
fake_serializer.number_of_calls += 1
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.calculator.do_means(self.sites, 1,
curve_serializer=lambda _: True,
curve_task=test_data_reflector)
self.assertEqual(0, fake_serializer.number_of_calls)
def test_map_serializer_not_called_unless_configured(self):
"""
The mean map serialization function is not called unless the
POES parameter was specified in the configuration file.
"""
def fake_serializer(kvs_keys):
"""Fake serialization function to be used in this test."""
# Check that the data returned is the one we expect for the current
# realization.
fake_serializer.number_of_calls += 1
fake_serializer.number_of_calls = 0
key = TestStore.put(self.job_ctxt.job_id, self.mock_results)
self.keys.append(key)
self.calculator.do_means(self.sites,
1,
curve_serializer=lambda _: True,
curve_task=test_data_reflector)
self.assertEqual(0, fake_serializer.number_of_calls)
def test_distribute_with_ignore_result_set_and_ath(self):
"""
The specified number of subtasks is actually spawned (even for tasks
with ignore_result=True) and the asynchronous task handler function is
run.
"""
def value(key):
"""Construct a test value for the given key."""
return key[-3:] * 2
def ath(data):
"""
An asynchronous task handler function that converts all task
results to upper case and returns the list of keys found.
"""
items_expected = len(data)
items_found = []
while len(items_found) < items_expected:
for key, _ in data:
if key in items_found:
continue
value = TestStore.get(key)
if value is not None:
TestStore.set(key, value.upper())
items_found.append(key)
time.sleep(0.05)
return items_found
keys = ["irtc:%s" % str(uuid.uuid4())[:8] for _ in xrange(5)]
values = [value(uid) for uid in keys]
data = zip(keys, values)
args = ("data", [[d] for d in data])
result = tasks.distribute(ignore_result,
args,
ath=ath,
ath_args=dict(data=data))
self.assertEqual(sorted(keys), sorted(result))
for key, value in data:
self.assertEqual(value.upper(), TestStore.get(key))
def test_distribute_with_ignore_result_set_and_ath(self):
"""
The specified number of subtasks is actually spawned (even for tasks
with ignore_result=True) and the asynchronous task handler function is
run.
"""
def value(key):
"""Construct a test value for the given key."""
return key[-3:] * 2
def ath(data):
"""
An asynchronous task handler function that converts all task
results to upper case and returns the list of keys found.
"""
items_expected = len(data)
items_found = []
while len(items_found) < items_expected:
for key, _ in data:
if key in items_found:
continue
value = TestStore.get(key)
if value is not None:
TestStore.set(key, value.upper())
items_found.append(key)
time.sleep(0.05)
return items_found
keys = ["irtc:%s" % str(uuid.uuid4())[:8] for _ in xrange(5)]
values = [value(uid) for uid in keys]
data = zip(keys, values)
args = ("data", [[d] for d in data])
result = tasks.distribute(ignore_result, args, ath=ath,
ath_args=dict(data=data))
self.assertEqual(sorted(keys), sorted(result))
for key, value in data:
self.assertEqual(value.upper(), TestStore.get(key))
def setUp(self):
# Make sure we have no obsolete test data in the kvs.
kvs = TestStore.kvs()
existing_data = kvs.keys("irtc:*")
if existing_data:
kvs.delete(*existing_data)
def tearDown(self):
# Remove the canned result data from the KVS.
for key in self.keys:
TestStore.remove(key)
def setUp(self):
# Make sure we have no obsolete test data in the kvs.
kvs = TestStore.kvs()
existing_data = kvs.keys("irtc:*")
if existing_data:
kvs.delete(*existing_data)
def tearDown(self):
# Remove the canned result data from the KVS.
for key in self.keys:
TestStore.remove(key)