def test_amqp_sanity(self):
"""We can talk to ourselves from Python using RabbitMQ"""
conn, ch, qname = self.setup_queue()
# now there is a queue, send a test message
sender = java.AMQPConnection()
sender.setHost(config.get("amqp", "host"))
sender.setPort(int(config.get("amqp", "port")))
sender.setUsername(config.get("amqp", "user"))
sender.setPassword(config.get("amqp", "password"))
sender.setVirtualHost(config.get("amqp", "vhost"))
sender.publish('oq-unittest.topic', 'oq-unittest-log.FOO', 0, 'WARN',
'Hi there')
sender.close()
# process the messaages
messages = []
def consume(msg):
messages.append(msg)
ch.basic_cancel(msg.consumer_tag)
self.consume_messages(conn, ch, qname, consume)
self.assertEquals(1, len(messages))
self.assertEquals('Hi there', messages[0].body)
def compute_uhs(the_job, site):
"""Given a `JobContext` and a site of interest, compute UHS. The Java
`UHSCalculator` is called to do perform the core computation.
:param the_job:
:class:`openquake.engine.JobContext` instance.
:param site:
:class:`openquake.shapes.Site` instance.
:returns:
An `ArrayList` (Java object) of `UHSResult` objects, one per PoE.
"""
periods = list_to_jdouble_array(the_job["UHS_PERIODS"])
poes = list_to_jdouble_array(the_job["POES"])
imls = get_iml_list(the_job["INTENSITY_MEASURE_LEVELS"], the_job["INTENSITY_MEASURE_TYPE"])
max_distance = the_job["MAXIMUM_DISTANCE"]
cache = java.jclass("KVS")(config.get("kvs", "host"), int(config.get("kvs", "port")))
erf = generate_erf(the_job.job_id, cache)
gmpe_map = generate_gmpe_map(the_job.job_id, cache)
set_gmpe_params(gmpe_map, the_job.params)
uhs_calc = java.jclass("UHSCalculator")(periods, poes, imls, erf, gmpe_map, max_distance)
uhs_results = uhs_calc.computeUHS(
site.latitude,
site.longitude,
the_job["VS30_TYPE"],
the_job["REFERENCE_VS30_VALUE"],
the_job["DEPTHTO1PT0KMPERSEC"],
the_job["REFERENCE_DEPTH_TO_2PT5KM_PER_SEC_PARAM"],
)
return uhs_results
def execute(self):
"""
Calculation work is parallelized over sources, which means that each
task will compute hazard for all sites but only with a subset of the
seismic sources defined in the input model.
The general workflow is as follows:
1. Fill the queue with an initial set of tasks. The number of initial
tasks is configurable using the `concurrent_tasks` parameter in the
`[hazard]` section of the OpenQuake config file.
2. Wait for tasks to signal completion (via AMQP message) and enqueue a
new task each time another completes. Once all of the job work is
enqueued, we just wait until all of the tasks conclude.
"""
block_size = int(config.get('hazard', 'block_size'))
concurrent_tasks = int(config.get('hazard', 'concurrent_tasks'))
self.progress = dict(total=0, computed=0)
# The following two counters are in a dict so that we can use them in
# the closures below.
# When `self.progress['compute']` becomes equal to
# `self.progress['total']`, # `execute` can conclude.
task_gen = self.task_arg_gen(block_size)
exchange, conn_args = exchange_and_conn_args()
routing_key = ROUTING_KEY_FMT % dict(job_id=self.job.id)
task_signal_queue = kombu.Queue(
'htasks.job.%s' % self.job.id, exchange=exchange,
routing_key=routing_key, durable=False, auto_delete=True)
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(
task_signal_queue,
callbacks=[self.get_task_complete_callback(task_gen)]):
# First: Queue up the initial tasks.
for _ in xrange(concurrent_tasks):
try:
self.core_calc_task.apply_async(task_gen.next())
except StopIteration:
# If we get a `StopIteration` here, that means we have
# a number of tasks < concurrent_tasks.
# This basically just means that we could be
# under-utilizing worker node resources.
break
while (self.progress['computed'] < self.progress['total']):
# This blocks until a message is received.
# Once we receive a completion signal, enqueue the next
# piece of work (if there's anything left to be done).
# (The `task_complete_callback` will handle additional
# queuing.)
conn.drain_events()
logs.log_progress("hazard calculation 100% complete", 2)
def connect(self, *args, **kwargs):
host = config.get("kvs", "host")
port = config.get("kvs", "port")
port = int(port) if port else 6379
stats_db = config.get("kvs", "stats_db")
stats_db = int(stats_db) if stats_db else 15
args = {"host": host, "port": port, "db": stats_db}
return redis.Redis(**args)
def connect(self, *args, **kwargs):
host = config.get("kvs", "host")
port = config.get("kvs", "port")
port = int(port) if port else 6379
stats_db = config.get("kvs", "stats_db")
stats_db = int(stats_db) if stats_db else 15
args = {"host": host, "port": port, "db": stats_db}
return redis.Redis(**args)
def __init__(self, host=config.get("kvs", "host"),
port=int(config.get("kvs", "port")),
**kwargs):
if not self.__dict__:
args = {"host": host,
"port": port,
"db": kwargs.get('db', 0)}
self.conn = redis.Redis(**args)
def _redis():
"""Return a connection to the redis store."""
host = config.get("kvs", "host")
port = config.get("kvs", "port")
port = int(port) if port else 6379
stats_db = config.get("kvs", "stats_db")
stats_db = int(stats_db) if stats_db else 15
args = {"host": host, "port": port, "db": stats_db}
return redis.Redis(**args)
def _redis():
"""Return a connection to the redis store."""
host = config.get("kvs", "host")
port = config.get("kvs", "port")
port = int(port) if port else 6379
stats_db = config.get("kvs", "stats_db")
stats_db = int(stats_db) if stats_db else DEFAULT_STATS_DB
args = {"host": host, "port": port, "db": stats_db}
return redis.Redis(**args)
def compute_disagg_matrix(job_id, site, poe, result_dir):
""" Compute a complete 5D Disaggregation matrix. This task leans heavily
on the DisaggregationCalculator (in the OpenQuake Java lib) to handle this
computation.
The 5D matrix returned from the java calculator will be saved to a file in
HDF5 format.
:param job_id: id of the job record in the KVS
:type job_id: `str`
:param site: a single site of interest
:type site: :class:`openquake.shapes.Site` instance`
:param poe: Probability of Exceedence
:type poe: `float`
:param result_dir: location for the Java code to write the matrix in an
HDF5 file (in a distributed environment, this should be the path of a
mounted NFS)
:returns: 2-tuple of (ground_motion_value, path_to_h5_matrix_file)
"""
the_job = job.Job.from_kvs(job_id)
lat_bin_lims = the_job[job_cfg.LAT_BIN_LIMITS]
lon_bin_lims = the_job[job_cfg.LON_BIN_LIMITS]
mag_bin_lims = the_job[job_cfg.MAG_BIN_LIMITS]
eps_bin_lims = the_job[job_cfg.EPS_BIN_LIMITS]
jd = list_to_jdouble_array
disagg_calc = java.jclass('DisaggregationCalculator')(
jd(lat_bin_lims), jd(lon_bin_lims),
jd(mag_bin_lims), jd(eps_bin_lims))
cache = java.jclass('KVS')(
config.get('kvs', 'host'),
int(config.get('kvs', 'port')))
erf = generate_erf(job_id, cache)
gmpe_map = generate_gmpe_map(job_id, cache)
set_gmpe_params(gmpe_map, the_job.params)
imls = get_iml_list(the_job['INTENSITY_MEASURE_LEVELS'],
the_job['INTENSITY_MEASURE_TYPE'])
vs30_type = the_job['VS30_TYPE']
vs30_value = the_job['REFERENCE_VS30_VALUE']
depth_to_1pt0 = the_job['DEPTHTO1PT0KMPERSEC']
depth_to_2pt5 = the_job['REFERENCE_DEPTH_TO_2PT5KM_PER_SEC_PARAM']
matrix_result = disagg_calc.computeMatrix(
site.latitude, site.longitude, erf, gmpe_map, poe, imls,
vs30_type, vs30_value, depth_to_1pt0, depth_to_2pt5)
matrix_path = save_5d_matrix_to_h5(result_dir,
numpy.array(matrix_result.getMatrix()))
return (matrix_result.getGMV(), matrix_path)
def preloader(self, *args, **kwargs):
"""Validate job"""
self.cache = java.jclass("KVS")(
config.get("kvs", "host"),
int(config.get("kvs", "port")))
self.calc = java.jclass("LogicTreeProcessor")(
self.cache, self.key)
java.jvm().java.lang.System.setProperty("openquake.nrml.schema",
xml.nrml_schema_file())
return fn(self, *args, **kwargs)
def decorated(self, *args, **kwargs): # pylint: disable=C0111
kvs_data = (config.get("kvs", "host"), int(config.get("kvs", "port")))
if kvs.cache_connections():
key = hashlib.md5(repr(kvs_data)).hexdigest()
if key not in __KVS_CONN_CACHE:
__KVS_CONN_CACHE[key] = java.jclass("KVS")(*kvs_data)
self.cache = __KVS_CONN_CACHE[key]
else:
self.cache = java.jclass("KVS")(*kvs_data)
return fn(self, *args, **kwargs)
def decorated(self, *args, **kwargs): # pylint: disable=C0111
kvs_data = (config.get("kvs", "host"), int(config.get("kvs", "port")))
if kvs.cache_connections():
key = hashlib.md5(repr(kvs_data)).hexdigest()
if key not in __KVS_CONN_CACHE:
__KVS_CONN_CACHE[key] = java.jclass("KVS")(*kvs_data)
self.cache = __KVS_CONN_CACHE[key]
else:
self.cache = java.jclass("KVS")(*kvs_data)
return fn(self, *args, **kwargs)
def setUp(self):
self.amqp = logs.AMQPHandler(
host=config.get("amqp", "host"),
username=config.get("amqp", "user"),
password=config.get("amqp", "password"),
virtual_host=config.get("amqp", "vhost"),
exchange='oq-unittest.topic',
routing_key='oq-unittest-log.%(levelname)s',
level=logging.DEBUG)
self.log = logging.getLogger('tests.PythonAMQPLogTestCase')
self.log.setLevel(logging.DEBUG)
self.log.addHandler(self.amqp)
def setup_queue(self):
# connect to localhost and bind to a queue
conn = amqp.Connection(host=config.get("amqp", "host"),
userid=config.get("amqp", "user"),
password=config.get("amqp", "password"),
virtual_host=config.get("amqp", "vhost"))
ch = conn.channel()
ch.access_request(config.get("amqp", "vhost"), active=False, read=True)
ch.exchange_declare(self.TOPIC, 'topic', auto_delete=True)
qname, _, _ = ch.queue_declare()
ch.queue_bind(qname, self.TOPIC, routing_key=self.ROUTING_KEY)
return conn, ch, qname
def setUp(self):
# starting the jvm...
print "About to start the jvm..."
jpype = java.jvm()
java_class = jpype.JClass("org.gem.engine.hazard.redis.Cache")
print "Not dead yet, and found the class..."
self.java_client = java_class(config.get("kvs", "host"),
int(config.get("kvs", "port")))
self.python_client = kvs.get_client()
self.python_client.flushdb()
self._delete_test_file()
def setUp(self):
# starting the jvm...
print "About to start the jvm..."
jpype = java.jvm()
java_class = jpype.JClass("org.gem.engine.hazard.redis.Cache")
print "Not dead yet, and found the class..."
self.java_client = java_class(
config.get("kvs", "host"), int(config.get("kvs", "port")))
self.python_client = kvs.get_client()
self.python_client.flushdb()
self._delete_test_file()
def compute_uhs(the_job, site):
"""Given a `JobContext` and a site of interest, compute UHS. The Java
`UHSCalculator` is called to do perform the core computation.
:param the_job:
:class:`openquake.engine.JobContext` instance.
:param site:
:class:`openquake.shapes.Site` instance.
:returns:
An `ArrayList` (Java object) of `UHSResult` objects, one per PoE.
"""
periods = list_to_jdouble_array(the_job['UHS_PERIODS'])
poes = list_to_jdouble_array(the_job['POES'])
imls = general.get_iml_list(the_job['INTENSITY_MEASURE_LEVELS'],
the_job['INTENSITY_MEASURE_TYPE'])
max_distance = the_job['MAXIMUM_DISTANCE']
cache = java.jclass('KVS')(
config.get('kvs', 'host'),
int(config.get('kvs', 'port')))
erf = general.generate_erf(the_job.job_id, cache)
gmpe_map = general.generate_gmpe_map(the_job.job_id, cache)
general.set_gmpe_params(gmpe_map, the_job.params)
uhs_calc = java.jclass('UHSCalculator')(periods, poes, imls, erf, gmpe_map,
max_distance)
site_model = general.get_site_model(the_job.oq_job.id)
if site_model is not None:
sm_data = general.get_closest_site_model_data(site_model, site)
vs30_type = sm_data.vs30_type.capitalize()
vs30 = sm_data.vs30
z1pt0 = sm_data.z1pt0
z2pt5 = sm_data.z2pt5
else:
jp = the_job.oq_job_profile
vs30_type = jp.vs30_type.capitalize()
vs30 = jp.reference_vs30_value
z1pt0 = jp.depth_to_1pt_0km_per_sec
z2pt5 = jp.reference_depth_to_2pt5km_per_sec_param
uhs_results = _compute_uhs(
uhs_calc, site.latitude, site.longitude, vs30_type, vs30, z1pt0, z2pt5
)
return uhs_results
def execute(self):
"""Main execution point for the Disaggregation calculator.
The workflow is structured like so:
1) Store source and GMPE models in the KVS (so the workers can rapidly
access that data).
2) Create a result dir (on the NFS) for storing matrices.
3) Distribute full disaggregation matrix computation to workers.
4) Distribute matrix subset extraction (using full disagg. results as
input.
5) Finally, write an NRML/XML wrapper around the disagg. results.
"""
# matrix results for this job will go here:
result_dir = DisaggHazardCalculator.create_result_dir(
config.get('nfs', 'base_dir'), self.job_ctxt.job_id)
realizations = self.job_ctxt['NUMBER_OF_LOGIC_TREE_SAMPLES']
poes = self.job_ctxt['POES']
sites = self.job_ctxt.sites_to_compute()
log_msg = ("Computing disaggregation for job_id=%s, %s sites, "
"%s realizations, and PoEs=%s")
log_msg %= (self.job_ctxt.job_id, len(sites), realizations, poes)
LOG.info(log_msg)
full_disagg_results = self.distribute_disagg(sites, realizations, poes,
result_dir)
subset_types = self.job_ctxt['DISAGGREGATION_RESULTS']
subset_results = self.distribute_subsets(full_disagg_results,
subset_types, result_dir)
DisaggHazardCalculator.serialize_nrml(self.job_ctxt, subset_types,
subset_results)
def test_get_with_empty_section_data(self):
"""config.get() returns `None` if the section data dict is empty."""
with patch('openquake.utils.config.get_section') as mock:
mock.return_value = dict()
self.assertTrue(config.get("whatever", "key") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("whatever", ), {}], mock.call_args)
def execute(self):
"""Main execution point for the Disaggregation calculator.
The workflow is structured like so:
1) Store source and GMPE models in the KVS (so the workers can rapidly
access that data).
2) Create a result dir (on the NFS) for storing matrices.
3) Distribute full disaggregation matrix computation to workers.
4) Distribute matrix subset extraction (using full disagg. results as
input.
5) Finally, write an NRML/XML wrapper around the disagg. results.
"""
# matrix results for this job will go here:
result_dir = DisaggHazardCalculator.create_result_dir(
config.get('nfs', 'base_dir'), self.calc_proxy.job_id)
realizations = self.calc_proxy['NUMBER_OF_LOGIC_TREE_SAMPLES']
poes = self.calc_proxy['POES']
sites = self.calc_proxy.sites_to_compute()
log_msg = ("Computing disaggregation for job_id=%s, %s sites, "
"%s realizations, and PoEs=%s")
log_msg %= (self.calc_proxy.job_id, len(sites), realizations, poes)
LOG.info(log_msg)
full_disagg_results = self.distribute_disagg(sites, realizations, poes,
result_dir)
subset_types = self.calc_proxy['DISAGGREGATION_RESULTS']
subset_results = self.distribute_subsets(full_disagg_results,
subset_types, result_dir)
DisaggHazardCalculator.serialize_nrml(self.calc_proxy, subset_types,
subset_results)
def test_get_with_empty_section_data(self):
"""config.get() returns `None` if the section data dict is empty."""
with patch('openquake.utils.config.get_section') as mock:
mock.return_value = dict()
self.assertTrue(config.get("whatever", "key") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("whatever",), {}], mock.call_args)
def test_get_with_unknown_key(self):
"""config.get() returns `None` if the `key` is not known."""
with patch('openquake.utils.config.get_section') as mock:
mock.return_value = dict(b=1)
self.assertTrue(config.get("arghh", "c") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("arghh",), {}], mock.call_args)
def record_init_stats(self):
"""
Record some basic job stats, including the number of sites,
realizations (end branches), and total number of tasks for the job.
This should be run between the `pre-execute` and `execute` phases, once
the job has been fully initialized.
"""
# Record num sites, num realizations, and num tasks.
num_sites = len(self.hc.points_to_compute())
realizations = models.LtRealization.objects.filter(
hazard_calculation=self.hc.id)
num_rlzs = realizations.count()
# Compute the number of tasks.
block_size = int(config.get('hazard', 'block_size'))
num_tasks = 0
for lt_rlz in realizations:
# Each realization has the potential to choose a random source
# model, and thus there may be a variable number of tasks for each
# realization (depending on the number of the sources in the model
# which was chosen for the realization).
num_sources = models.SourceProgress.objects.filter(
lt_realization=lt_rlz).count()
num_tasks += math.ceil(float(num_sources) / block_size)
models.JobStats.objects.filter(oq_job=self.job.id).update(
num_sites=num_sites, num_tasks=num_tasks,
num_realizations=num_rlzs)
def test_get_with_unknown_key(self):
"""config.get() returns `None` if the `key` is not known."""
with patch('openquake.utils.config.get_section') as mock:
mock.return_value = dict(b=1)
self.assertTrue(config.get("arghh", "c") is None)
self.assertEqual(1, mock.call_count)
self.assertEqual([("arghh", ), {}], mock.call_args)
def test_get_with_nonempty_section_data_and_known_key(self):
"""
config.get() correctly returns the configuration datum for known
sections/keys.
"""
with patch('openquake.utils.config.get_section') as mock:
mock.return_value = dict(a=11)
self.assertEqual(11, config.get("hmmm", "a"))
self.assertEqual(1, mock.call_count)
self.assertEqual([("hmmm", ), {}], mock.call_args)
def from_kvs(job_id):
"""Return the job in the underlying kvs system with the given id."""
logs.init_logs(
level=FLAGS.debug, log_type=oq_config.get("logging", "backend"))
params = kvs.get_value_json_decoded(
kvs.tokens.generate_job_key(job_id))
job = Job(params, job_id)
return job
def test_get_with_nonempty_section_data_and_known_key(self):
"""
config.get() correctly returns the configuration datum for known
sections/keys.
"""
with patch('openquake.utils.config.get_section') as mock:
mock.return_value = dict(a=11)
self.assertEqual(11, config.get("hmmm", "a"))
self.assertEqual(1, mock.call_count)
self.assertEqual([("hmmm",), {}], mock.call_args)
def setUp(self):
jvm = java.jvm()
props = jvm.JClass("java.util.Properties")()
props.setProperty('log4j.rootLogger', 'DEBUG, rabbit')
for key, value in [
('', 'org.gem.log.AMQPAppender'),
('.host', config.get("amqp", "host")),
('.port', config.get("amqp", "port")),
('.username', config.get("amqp", "user")),
('.password', config.get("amqp", "password")),
('.virtualHost', config.get("amqp", "vhost")),
('.routingKeyPattern', 'oq-unittest-log.%p'),
('.exchange', 'oq-unittest.topic'),
('.layout', 'org.apache.log4j.PatternLayout'),
('.layout.ConversionPattern', '%p - %m')]:
props.setProperty('log4j.appender.rabbit' + key, value)
jvm.JClass("org.apache.log4j.BasicConfigurator").resetConfiguration()
jvm.JClass("org.apache.log4j.PropertyConfigurator").configure(props)
def compute_uhs(the_job, site):
"""Given a `CalculationProxy` and a site of interest, compute UHS. The Java
`UHSCalculator` is called to do perform the core computation.
:param the_job:
:class:`openquake.engine.CalculationProxy` instance.
:param site:
:class:`openquake.shapes.Site` instance.
:returns:
An `ArrayList` (Java object) of `UHSResult` objects, one per PoE.
"""
periods = list_to_jdouble_array(the_job['UHS_PERIODS'])
poes = list_to_jdouble_array(the_job['POES'])
imls = get_iml_list(the_job['INTENSITY_MEASURE_LEVELS'],
the_job['INTENSITY_MEASURE_TYPE'])
max_distance = the_job['MAXIMUM_DISTANCE']
cache = java.jclass('KVS')(
config.get('kvs', 'host'),
int(config.get('kvs', 'port')))
erf = generate_erf(the_job.job_id, cache)
gmpe_map = generate_gmpe_map(the_job.job_id, cache)
set_gmpe_params(gmpe_map, the_job.params)
uhs_calc = java.jclass('UHSCalculator')(periods, poes, imls, erf, gmpe_map,
max_distance)
uhs_results = uhs_calc.computeUHS(
site.latitude,
site.longitude,
the_job['VS30_TYPE'],
the_job['REFERENCE_VS30_VALUE'],
the_job['DEPTHTO1PT0KMPERSEC'],
the_job['REFERENCE_DEPTH_TO_2PT5KM_PER_SEC_PARAM'])
return uhs_results
def spawn_job_supervisor(job_id, pid):
"""
Spawn a supervisor process as configured in openquake.cfg.
:param int job_id: the id of the job to be supervised
:param int pid: the process id of the job to be supervised
:return: the id of the supervisor process or None if no supervisor was
configured
:rtype: int or None
"""
exe = oq_config.get('supervisor', 'exe')
if exe:
if oq_config.get('logging', 'backend') != 'amqp':
LOG.warn('If you want to run supervised jobs it\'s better '
'to set [logging] backend=amqp in openquake.cfg')
if not os.path.isabs(exe):
exe = os.path.join(OPENQUAKE_ROOT, exe)
cmd = [exe, str(job_id), str(pid)]
supervisor_pid = subprocess.Popen(cmd, env=os.environ).pid
job = OqJob.objects.get(id=job_id)
job.supervisor_pid = supervisor_pid
job.job_pid = pid
job.save()
# Ensure the supervisor amqp queue exists
supervisor.bind_supervisor_queue(job_id)
return supervisor_pid
else:
LOG.warn('This job won\'t be supervised, '
'because no supervisor is configured in openquake.cfg')
def get_client(**kwargs):
"""
Return a redis kvs client connection for general OpenQuake engine
calculation usage..
PLEASE NOTE: The 'db' argument is automatically read from the openquake.cfg
and set. If specified in ``kwargs``, it will be overridden with the setting
in openquake.cfg.
"""
global __KVS_CONN_POOL
if __KVS_CONN_POOL is None:
cfg = config.get_section("kvs")
# get the default db from the openquake.cfg:
db = int(config.get('kvs', 'redis_db'))
__KVS_CONN_POOL = redis.ConnectionPool(
max_connections=1, host=cfg["host"], port=int(cfg["port"]), db=db)
kwargs.update({"connection_pool": __KVS_CONN_POOL})
return redis.Redis(**kwargs)
def jvm():
"""Return the jpype module, after guaranteeing the JVM is running and
the classpath has been loaded properly."""
jarpaths = (os.path.abspath(
os.path.join(os.path.dirname(__file__), "../dist")),
'/usr/share/java')
if not jpype.isJVMStarted():
jpype.startJVM(jpype.getDefaultJVMPath(),
"-Djava.ext.dirs=%s:%s" % jarpaths,
# force the default Xerces parser configuration, otherwise
# some random system-installed JAR might override it
"-Dorg.apache.xerces.xni.parser.XMLParserConfiguration=" \
"org.apache.xerces.parsers.XIncludeAwareParserConfiguration")
init_logs(level=FLAGS.debug, log_type=config.get("logging", "backend"))
return jpype
def do_hazard_map_post_process(job):
"""
Create and distribute tasks for processing hazard curves into hazard maps.
:param job:
A :class:`openquake.db.models.OqJob` which has some hazard curves
associated with it.
"""
logs.LOG.debug('> Post-processing - Hazard Maps')
block_size = int(config.get('hazard', 'concurrent_tasks'))
poes = job.hazard_calculation.poes_hazard_maps
# Stats for debug logging:
hazard_curve_ids = models.HazardCurve.objects.filter(
output__oq_job=job).values_list('id', flat=True)
logs.LOG.debug('num haz curves: %s' % len(hazard_curve_ids))
# Limit the number of concurrent tasks to the configured concurrency level:
block_gen = block_splitter(hazard_curve_ids, block_size)
total_blocks = int(math.ceil(len(hazard_curve_ids) / float(block_size)))
for i, block in enumerate(block_gen):
logs.LOG.debug('> Hazard post-processing block, %s of %s'
% (i + 1, total_blocks))
tasks = []
for hazard_curve_id in block:
tasks.append(hazard_curves_to_hazard_map_task.subtask(
(job.id, hazard_curve_id, poes)))
results = TaskSet(tasks=tasks).apply_async()
utils_tasks._check_exception(results)
logs.LOG.debug('< Done Hazard Map post-processing block, %s of %s'
% (i + 1, total_blocks))
logs.LOG.debug('< Done post-processing - Hazard Maps')
def compute_disagg_matrix(job_ctxt, site, poe, result_dir):
""" Compute a complete 5D Disaggregation matrix. This task leans heavily
on the DisaggregationCalculator (in the OpenQuake Java lib) to handle this
computation.
The 5D matrix returned from the java calculator will be saved to a file in
HDF5 format.
:param job_ctxt:
A :class:`openquake.engine.JobContext` which holds all of the
data we need to run this computation.
:param site: a single site of interest
:type site: :class:`openquake.shapes.Site` instance`
:param poe: Probability of Exceedence
:type poe: `float`
:param result_dir: location for the Java code to write the matrix in an
HDF5 file (in a distributed environment, this should be the path of a
mounted NFS)
:returns: 2-tuple of (ground_motion_value, path_to_h5_matrix_file)
"""
lat_bin_lims = job_ctxt[job_cfg.LAT_BIN_LIMITS]
lon_bin_lims = job_ctxt[job_cfg.LON_BIN_LIMITS]
mag_bin_lims = job_ctxt[job_cfg.MAG_BIN_LIMITS]
eps_bin_lims = job_ctxt[job_cfg.EPS_BIN_LIMITS]
jd = list_to_jdouble_array
disagg_calc = java.jclass('DisaggregationCalculator')(
jd(lat_bin_lims), jd(lon_bin_lims),
jd(mag_bin_lims), jd(eps_bin_lims))
cache = java.jclass('KVS')(
config.get('kvs', 'host'),
int(config.get('kvs', 'port')))
erf = general.generate_erf(job_ctxt.job_id, cache)
gmpe_map = general.generate_gmpe_map(job_ctxt.job_id, cache)
general.set_gmpe_params(gmpe_map, job_ctxt.params)
imls = general.get_iml_list(job_ctxt['INTENSITY_MEASURE_LEVELS'],
job_ctxt['INTENSITY_MEASURE_TYPE'])
site_model = general.get_site_model(job_ctxt.oq_job.id)
if site_model is not None:
sm_data = general.get_closest_site_model_data(site_model, site)
vs30_type = sm_data.vs30_type.capitalize()
vs30 = sm_data.vs30
z1pt0 = sm_data.z1pt0
z2pt5 = sm_data.z2pt5
else:
jp = job_ctxt.oq_job_profile
vs30_type = jp.vs30_type.capitalize()
vs30 = jp.reference_vs30_value
z1pt0 = jp.depth_to_1pt_0km_per_sec
z2pt5 = jp.reference_depth_to_2pt5km_per_sec_param
matrix_result = _compute_matrix(
disagg_calc, site.latitude, site.longitude, erf, gmpe_map, poe, imls,
vs30_type, vs30, z1pt0, z2pt5)
matrix_path = save_5d_matrix_to_h5(result_dir,
numpy.array(matrix_result.getMatrix()))
return (matrix_result.getGMV(), matrix_path)
def __new__(cls, host=config.get("kvs", "host"),
port=int(config.get("kvs", "port")),
**kwargs): # pylint: disable=W0613
self = object.__new__(cls)
self.__dict__ = cls.__shared_state
return self
def execute(self):
"""
Calculation work is parallelized over sources, which means that each
task will compute hazard for all sites but only with a subset of the
seismic sources defined in the input model.
The general workflow is as follows:
1. Fill the queue with an initial set of tasks. The number of initial
tasks is configurable using the `concurrent_tasks` parameter in the
`[hazard]` section of the OpenQuake config file.
2. Wait for tasks to signal completion (via AMQP message) and enqueue a
new task each time another completes. Once all of the job work is
enqueued, we just wait until all of the tasks conclude.
"""
job = self.job
hc = job.hazard_calculation
sources_per_task = int(config.get('hazard', 'block_size'))
concurrent_tasks = int(config.get('hazard', 'concurrent_tasks'))
progress = dict(total=0, computed=0)
# The following two counters are in a dict so that we can use them in
# the closures below.
# When `progress['compute']` becomes equal to `progress['total']`,
# `execute` can conclude.
task_gen = self.task_arg_gen(hc, job, sources_per_task, progress)
def task_complete_callback(body, message):
"""
:param dict body:
``body`` is the message sent by the task. The dict should
contain 2 keys: `job_id` and `num_sources` (to indicate the
number of sources computed).
Both values are `int`.
:param message:
A :class:`kombu.transport.pyamqplib.Message`, which contains
metadata about the message (including content type, channel,
etc.). See kombu docs for more details.
"""
job_id = body['job_id']
num_sources = body['num_sources']
assert job_id == job.id
progress['computed'] += num_sources
logs.log_percent_complete(job_id, "hazard")
# Once we receive a completion signal, enqueue the next
# piece of work (if there's anything left to be done).
try:
self.core_calc_task.apply_async(task_gen.next())
except StopIteration:
# There are no more tasks to dispatch; now we just need
# to wait until all tasks signal completion.
pass
message.ack()
exchange, conn_args = exchange_and_conn_args()
routing_key = ROUTING_KEY_FMT % dict(job_id=job.id)
task_signal_queue = kombu.Queue(
'htasks.job.%s' % job.id, exchange=exchange,
routing_key=routing_key, durable=False, auto_delete=True)
with kombu.BrokerConnection(**conn_args) as conn:
task_signal_queue(conn.channel()).declare()
with conn.Consumer(task_signal_queue,
callbacks=[task_complete_callback]):
# First: Queue up the initial tasks.
for _ in xrange(concurrent_tasks):
try:
self.core_calc_task.apply_async(task_gen.next())
except StopIteration:
# If we get a `StopIteration` here, that means we have
# a number of tasks < concurrent_tasks.
# This basically just means that we could be
# under-utilizing worker node resources.
break
while (progress['computed'] < progress['total']):
# This blocks until a message is received.
# Once we receive a completion signal, enqueue the next
# piece of work (if there's anything left to be done).
# (The `task_complete_callback` will handle additional
# queuing.)
conn.drain_events()
logs.log_progress("hazard calculation 100% complete", 2)
def open():
"""Initialize the test store."""
if TestStore._conn is not None:
return
TestStore._conn = redis.Redis(db=int(config.get("kvs", "test_db")))
def compute_disagg_matrix(job_ctxt, site, poe, result_dir):
""" Compute a complete 5D Disaggregation matrix. This task leans heavily
on the DisaggregationCalculator (in the OpenQuake Java lib) to handle this
computation.
The 5D matrix returned from the java calculator will be saved to a file in
HDF5 format.
:param job_ctxt:
A :class:`openquake.engine.JobContext` which holds all of the
data we need to run this computation.
:param site: a single site of interest
:type site: :class:`openquake.shapes.Site` instance`
:param poe: Probability of Exceedence
:type poe: `float`
:param result_dir: location for the Java code to write the matrix in an
HDF5 file (in a distributed environment, this should be the path of a
mounted NFS)
:returns: 2-tuple of (ground_motion_value, path_to_h5_matrix_file)
"""
lat_bin_lims = job_ctxt[job_cfg.LAT_BIN_LIMITS]
lon_bin_lims = job_ctxt[job_cfg.LON_BIN_LIMITS]
mag_bin_lims = job_ctxt[job_cfg.MAG_BIN_LIMITS]
eps_bin_lims = job_ctxt[job_cfg.EPS_BIN_LIMITS]
jd = list_to_jdouble_array
disagg_calc = java.jclass('DisaggregationCalculator')(jd(lat_bin_lims),
jd(lon_bin_lims),
jd(mag_bin_lims),
jd(eps_bin_lims))
cache = java.jclass('KVS')(config.get('kvs', 'host'),
int(config.get('kvs', 'port')))
erf = general.generate_erf(job_ctxt.job_id, cache)
gmpe_map = general.generate_gmpe_map(job_ctxt.job_id, cache)
general.set_gmpe_params(gmpe_map, job_ctxt.params)
imls = general.get_iml_list(job_ctxt['INTENSITY_MEASURE_LEVELS'],
job_ctxt['INTENSITY_MEASURE_TYPE'])
site_model = general.get_site_model(job_ctxt.oq_job.id)
if site_model is not None:
sm_data = general.get_closest_site_model_data(site_model, site)
vs30_type = sm_data.vs30_type.capitalize()
vs30 = sm_data.vs30
z1pt0 = sm_data.z1pt0
z2pt5 = sm_data.z2pt5
else:
jp = job_ctxt.oq_job_profile
vs30_type = jp.vs30_type.capitalize()
vs30 = jp.reference_vs30_value
z1pt0 = jp.depth_to_1pt_0km_per_sec
z2pt5 = jp.reference_depth_to_2pt5km_per_sec_param
matrix_result = _compute_matrix(disagg_calc, site.latitude, site.longitude,
erf, gmpe_map, poe, imls, vs30_type, vs30,
z1pt0, z2pt5)
matrix_path = save_5d_matrix_to_h5(result_dir,
numpy.array(matrix_result.getMatrix()))
return (matrix_result.getGMV(), matrix_path)
LOG4J_STDOUT_FORMAT = '%-5p %X{processName} [%c] - Job %X{job_id} - %m%n'
LOG4J_STDOUT_SETTINGS = {
'log4j.rootLogger': '%(level)s, stdout',
'log4j.appender.stdout': 'org.apache.log4j.ConsoleAppender',
'log4j.appender.stdout.follow': 'true',
'log4j.appender.stdout.layout': 'org.apache.log4j.PatternLayout',
'log4j.appender.stdout.layout.ConversionPattern': LOG4J_STDOUT_FORMAT,
}
LOG4J_AMQP_SETTINGS = {
'log4j.rootLogger': '%(level)s, amqp',
'log4j.appender.amqp': 'org.gem.log.AMQPAppender',
'log4j.appender.amqp.host': config.get("amqp", "host"),
'log4j.appender.amqp.port': config.get("amqp", "port"),
'log4j.appender.amqp.username': config.get("amqp", "user"),
'log4j.appender.amqp.password': config.get("amqp", "password"),
'log4j.appender.amqp.virtualHost': config.get("amqp", "vhost"),
'log4j.appender.amqp.routingKeyPattern': 'log.%p.%X{job_id}',
'log4j.appender.amqp.exchange': config.get("amqp", "exchange"),
'log4j.appender.amqp.layout': 'org.apache.log4j.PatternLayout',
'log4j.appender.amqp.layout.ConversionPattern': LOG4J_AMQP_FORMAT,
}
def init_logs(log_type='console', level='warn'):
"""
Initialize Python logging.
def open():
"""Initialize the test store."""
if TestStore._conn is not None:
return
TestStore._conn = redis.Redis(db=int(config.get("kvs", "test_db")))
