def __init__(self, calc_id):
[smlt_input] = models.inputs4hcalc(calc_id, input_type='lt_source')
smlt_content = smlt_input.model_content.raw_content_ascii
[gmpelt_input] = models.inputs4hcalc(calc_id, input_type='lt_gsim')
gmpelt_content = gmpelt_input.model_content.raw_content_ascii
self.source_model_lt = SourceModelLogicTree(
smlt_content, basepath=None, filename=None, validate=False
)
self.gmpe_lt = GMPELogicTree(
tectonic_region_types=[], content=gmpelt_content,
basepath=None, filename=None, validate=False
)
def initialize_sources(self):
"""
Parse and validation logic trees (source and gsim). Then get all
sources referenced in the the source model logic tree, create
:class:`~openquake.engine.db.models.Input` records for all of them,
parse then, and save the parsed sources to the `parsed_source` table
(see :class:`openquake.engine.db.models.ParsedSource`).
"""
logs.LOG.progress("initializing sources")
[smlt] = models.inputs4hcalc(
self.hc.id, input_type='source_model_logic_tree')
[gsimlt] = models.inputs4hcalc(
self.hc.id, input_type='gsim_logic_tree')
source_paths = logictree.read_logic_trees(
self.hc.base_path, smlt.path, gsimlt.path)
src_inputs = []
for src_path in source_paths:
full_path = os.path.join(self.hc.base_path, src_path)
# Get or reuse the 'source' Input:
inp = engine2.get_input(
full_path, 'source', self.hc.owner, self.hc.force_inputs)
src_inputs.append(inp)
# Associate the source input to the calculation:
models.Input2hcalc.objects.get_or_create(
input=inp, hazard_calculation=self.hc)
# Associate the source input to the source model logic tree input:
try:
models.Src2ltsrc.objects.get(lt_src=smlt, filename=src_path)
except ObjectDoesNotExist:
# If it doesn't exist, this is a new input and we're not
# reusing an old one which is identical.
# Only in this case do we parse the sources and populate
# `hzrdi.parsed_source`.
models.Src2ltsrc.objects.create(hzrd_src=inp, lt_src=smlt,
filename=src_path)
src_content = StringIO.StringIO(
inp.model_content.raw_content_ascii)
sm_parser = nrml_parsers.SourceModelParser(src_content)
src_db_writer = source.SourceDBWriter(
inp, sm_parser.parse(), self.hc.rupture_mesh_spacing,
self.hc.width_of_mfd_bin,
self.hc.area_source_discretization
)
src_db_writer.serialize()
def task_arg_gen(self, block_size):
"""
Loop through realizations and sources to generate a sequence of
task arg tuples. Each tuple of args applies to a single task.
Yielded results are quadruples of (job_id, task_no,
rupture_id, random_seed). (random_seed will be used to seed
numpy for temporal occurence sampling.)
:param int block_size:
The number of work items for each task. Fixed to 1.
"""
rnd = random.Random()
rnd.seed(self.hc.random_seed)
inp = models.inputs4hcalc(self.hc.id, 'rupture_model')[0]
ruptures = models.ParsedRupture.objects.filter(input__id=inp.id)
rupture_ids = [rupture.id for rupture in ruptures]
num_concurrent_tasks = self.concurrent_tasks()
spare_realizations, realiz_per_task = gmf_realiz_per_task(
self.hc.number_of_ground_motion_fields, num_concurrent_tasks)
num_tasks = (num_concurrent_tasks + 1 if spare_realizations
else num_concurrent_tasks)
for task_no in range(num_tasks):
task_seed = rnd.randint(0, MAX_SINT_32)
task_args = (self.job.id, rupture_ids,
self.output.id, task_seed, task_no,
realiz_per_task[task_no])
yield task_args
def test_initialize_site_model(self):
# we need a slightly different config file for this test
cfg = helpers.demo_file(
'simple_fault_demo_hazard/job_with_site_model.ini')
self.job = helpers.get_hazard_job(cfg)
self.calc = core.ClassicalHazardCalculator(self.job)
self.calc.initialize_site_model()
# If the site model isn't valid for the calculation geometry, a
# `RuntimeError` should be raised here
# Okay, it's all good. Now check the count of the site model records.
[site_model_inp] = models.inputs4hcalc(
self.job.hazard_calculation.id, input_type='site_model')
sm_nodes = models.SiteModel.objects.filter(input=site_model_inp)
self.assertEqual(2601, len(sm_nodes))
num_pts_to_compute = len(
self.job.hazard_calculation.points_to_compute())
[site_data] = models.SiteData.objects.filter(
hazard_calculation=self.job.hazard_calculation.id)
# The site model is good. Now test that `site_data` was computed.
# For now, just test the lengths of the site data collections:
self.assertEqual(num_pts_to_compute, len(site_data.lons))
self.assertEqual(num_pts_to_compute, len(site_data.lats))
self.assertEqual(num_pts_to_compute, len(site_data.vs30s))
self.assertEqual(num_pts_to_compute, len(site_data.vs30_measured))
self.assertEqual(num_pts_to_compute, len(site_data.z1pt0s))
self.assertEqual(num_pts_to_compute, len(site_data.z2pt5s))
def _initialize_realizations_montecarlo(self, rlz_callbacks=None):
"""
Perform random sampling of both logic trees and populate lt_realization
table.
:param rlz_callbacks:
See :meth:`initialize_realizations` for more info.
"""
# Each realization will have two seeds:
# One for source model logic tree, one for GSIM logic tree.
rnd = random.Random()
seed = self.hc.random_seed
rnd.seed(seed)
[smlt] = models.inputs4hcalc(self.hc.id, input_type="lt_source")
ltp = logictree.LogicTreeProcessor(self.hc.id)
hzrd_src_cache = {}
# The first realization gets the seed we specified in the config file.
for i in xrange(self.hc.number_of_logic_tree_samples):
# Sample source model logic tree branch paths:
sm_name, sm_lt_path = ltp.sample_source_model_logictree(rnd.randint(MIN_SINT_32, MAX_SINT_32))
# Sample GSIM logic tree branch paths:
gsim_lt_path = ltp.sample_gmpe_logictree(rnd.randint(MIN_SINT_32, MAX_SINT_32))
lt_rlz = models.LtRealization(
hazard_calculation=self.hc,
ordinal=i,
seed=seed,
weight=None,
sm_lt_path=sm_lt_path,
gsim_lt_path=gsim_lt_path,
# we will update total_items in initialize_source_progress()
total_items=-1,
)
lt_rlz.save()
if not sm_name in hzrd_src_cache:
# Get the source model for this sample:
hzrd_src = models.Src2ltsrc.objects.get(lt_src=smlt.id, filename=sm_name).hzrd_src
# and cache it
hzrd_src_cache[sm_name] = hzrd_src
else:
hzrd_src = hzrd_src_cache[sm_name]
# Create source_progress objects
self.initialize_source_progress(lt_rlz, hzrd_src)
# Run realization callback (if any) to do additional initialization
# for each realization:
if rlz_callbacks is not None:
for cb in rlz_callbacks:
cb(lt_rlz)
# update the seed for the next realization
seed = rnd.randint(MIN_SINT_32, MAX_SINT_32)
rnd.seed(seed)
def test_a_few_inputs(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(
owner.user_name, params, files
)
inputs = models.inputs4hcalc(hc.id)
# We expect 3: the two logic trees and one source model
self.assertEqual(3, inputs.count())
def test_initalize_sources(self):
self.calc.initialize_sources()
# The source model logic tree for this configuration has only 1 source
# model:
[source] = models.inputs4hcalc(
self.job.hazard_calculation.id, input_type='source')
parsed_sources = models.ParsedSource.objects.filter(input=source)
# This source model contains 118 sources:
self.assertEqual(118, len(parsed_sources))
# Finally, check the Src2ltsrc linkage:
[smlt] = models.inputs4hcalc(
self.job.hazard_calculation.id, input_type='source_model_logic_tree')
[src2ltsrc] = models.Src2ltsrc.objects.filter(
hzrd_src=source, lt_src=smlt)
# Make sure the `filename` is exactly as it apprears in the logic tree.
# This is import for the logic tree processing we need to do later on.
self.assertEqual('dissFaultModel.xml', src2ltsrc.filename)
def test_with_input_type(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(
owner.user_name, params, files
)
inputs = models.inputs4hcalc(
hc.id, input_type='source_model_logic_tree'
)
self.assertEqual(1, inputs.count())
def initialize_sources(self):
"""
Parse and validation logic trees (source and gsim). Then get all
sources referenced in the the source model logic tree, create
:class:`~openquake.engine.db.models.Input` records for all of them,
parse then, and save the parsed sources to the `parsed_source` table
(see :class:`openquake.engine.db.models.ParsedSource`).
"""
logs.LOG.progress("initializing sources")
[smlt] = models.inputs4hcalc(
self.hc.id, input_type='source_model_logic_tree')
[gsimlt] = models.inputs4hcalc(
self.hc.id, input_type='gsim_logic_tree')
source_paths = logictree.read_logic_trees(
self.hc.base_path, smlt.path, gsimlt.path)
for src_path in source_paths:
full_path = os.path.join(self.hc.base_path, src_path)
# Get the 'source' Input:
inp = engine.get_input(full_path, 'source', self.hc.owner)
# Associate the source input to the calculation:
models.Input2hcalc.objects.get_or_create(
input=inp, hazard_calculation=self.hc)
models.Src2ltsrc.objects.create(hzrd_src=inp, lt_src=smlt,
filename=src_path)
src_content = StringIO.StringIO(
inp.model_content.raw_content_ascii)
sm_parser = nrml_parsers.SourceModelParser(src_content)
# Covert
src_db_writer = source.SourceDBWriter(
inp, sm_parser.parse(), self.hc.rupture_mesh_spacing,
self.hc.width_of_mfd_bin,
self.hc.area_source_discretization
)
src_db_writer.serialize()
def test_a_few_inputs(self):
cfg = helpers.demo_file('simple_fault_demo_hazard/job.ini')
params, files = engine2.parse_config(open(cfg, 'r'), force_inputs=True)
owner = helpers.default_user()
hc = engine2.create_hazard_calculation(owner, params, files.values())
expected_ids = sorted([x.id for x in files.values()])
inputs = models.inputs4hcalc(hc.id)
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def test_with_input_type(self):
cfg = helpers.demo_file("simple_fault_demo_hazard/job.ini")
params, files = engine2.parse_config(open(cfg, "r"), force_inputs=True)
owner = helpers.default_user()
hc = engine2.create_hazard_calculation(owner, params, files.values())
# It should only be 1 id, actually.
expected_ids = [x.id for x in files.values() if x.input_type == "lt_source"]
inputs = models.inputs4hcalc(hc.id, input_type="lt_source")
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def test_with_input_type(self):
cfg = helpers.get_data_path('simple_fault_demo_hazard/job.ini')
params, files = engine.parse_config(open(cfg, 'r'))
owner = helpers.default_user()
hc = engine.create_hazard_calculation(owner, params, files.values())
# It should only be 1 id, actually.
expected_ids = [x.id for x in files.values()
if x.input_type == 'source_model_logic_tree']
inputs = models.inputs4hcalc(
hc.id, input_type='source_model_logic_tree'
)
actual_ids = sorted([x.id for x in inputs])
self.assertEqual(expected_ids, actual_ids)
def _initialize_realizations_enumeration(self, rlz_callbacks=None):
"""
Perform full paths enumeration of logic trees and populate
lt_realization table.
:param rlz_callbacks:
See :meth:`initialize_realizations` for more info.
"""
hc = self.job.hazard_calculation
[smlt] = models.inputs4hcalc(hc.id,
input_type='source_model_logic_tree')
ltp = logictree.LogicTreeProcessor(hc.id)
hzrd_src_cache = {}
for i, path_info in enumerate(ltp.enumerate_paths()):
sm_name, weight, sm_lt_path, gsim_lt_path = path_info
lt_rlz = models.LtRealization(
hazard_calculation=hc,
ordinal=i,
seed=None,
weight=weight,
sm_lt_path=sm_lt_path,
gsim_lt_path=gsim_lt_path,
# we will update total_items in initialize_source_progress()
total_items=-1)
lt_rlz.save()
if not sm_name in hzrd_src_cache:
# Get the source model for this sample:
hzrd_src = models.Src2ltsrc.objects.get(
lt_src=smlt.id, filename=sm_name).hzrd_src
# and cache it
hzrd_src_cache[sm_name] = hzrd_src
else:
hzrd_src = hzrd_src_cache[sm_name]
# Create source_progress objects
self.initialize_source_progress(lt_rlz, hzrd_src)
# Run realization callback (if any) to do additional initialization
# for each realization:
if rlz_callbacks is not None:
for cb in rlz_callbacks:
cb(lt_rlz)
def initialize_sources(self):
"""
Get the rupture_model file from the job.ini file, and store a
parsed version of it in the database (see
:class:`openquake.engine.db.models.ParsedRupture``) in pickle format.
"""
# Get the rupture model in input
[inp] = models.inputs4hcalc(self.hc.id, input_type='rupture_model')
# Associate the source input to the calculation:
models.Input2hcalc.objects.get_or_create(
input=inp, hazard_calculation=self.hc)
# Store the ParsedRupture record
src_content = StringIO(inp.model_content.raw_content)
rupt_parser = RuptureModelParser(src_content)
src_db_writer = source.RuptureDBWriter(inp, rupt_parser.parse())
src_db_writer.serialize()
def get_site_model(hc_id):
"""Get the site model :class:`~openquake.engine.db.models.Input` record
for the given job id.
:param int hc_id:
The id of a :class:`~openquake.engine.db.models.HazardCalculation`.
:returns:
The site model :class:`~openquake.engine.db.models.Input` record for
this job.
:raises:
:exc:`RuntimeError` if the job has more than 1 site model.
"""
site_model = models.inputs4hcalc(hc_id, input_type="site_model")
if len(site_model) == 0:
return None
elif len(site_model) > 1:
# Multiple site models for 1 job are not allowed.
raise RuntimeError("Only 1 site model per job is allowed, found %s." % len(site_model))
# There's only one site model.
return site_model[0]
def task_arg_gen(self, block_size):
"""
Loop through realizations and sources to generate a sequence of
task arg tuples. Each tuple of args applies to a single task.
Yielded results are 6-uples of the form (job_id,
sites, rupture_id, output_id, task_seed, realizations)
(task_seed will be used to seed numpy for temporal occurence sampling).
:param int block_size:
The number of work items for each task. Fixed to 1.
"""
rnd = random.Random()
rnd.seed(self.hc.random_seed)
inp = models.inputs4hcalc(self.hc.id, 'rupture_model')[0]
ruptures = models.ParsedRupture.objects.filter(input__id=inp.id)
rupture_id = [rupture.id for rupture in ruptures][0] # only one
for sites in block_splitter(self.hc.site_collection, BLOCK_SIZE):
task_seed = rnd.randint(0, MAX_SINT_32)
yield (self.job.id, SiteCollection(sites),
rupture_id, self.output.id, task_seed,
self.hc.number_of_ground_motion_fields)
def test_no_inputs(self):
self.assertEqual([], list(models.inputs4hcalc(-1)))