Example #1
0
 def gen_args(self, ruptures_by_grp):
     """
     :param ruptures_by_grp: a dictionary of EBRupture objects
     :yields: the arguments for compute_gmfs_and_curves
     """
     oq = self.oqparam
     monitor = self.monitor(self.core_task.__name__)
     imts = list(oq.imtls)
     min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
     correl_model = oq.get_correl_model()
     try:
         csm_info = self.csm.info
     except AttributeError:  # no csm
         csm_info = self.datastore['csm_info']
     samples_by_grp = csm_info.get_samples_by_grp()
     for grp_id in ruptures_by_grp:
         ruptures = ruptures_by_grp[grp_id]
         if not ruptures:
             continue
         rlzs_by_gsim = self.rlzs_assoc.get_rlzs_by_gsim(grp_id)
         for block in block_splitter(ruptures, oq.ruptures_per_block):
             samples = samples_by_grp[grp_id]
             getter = GmfGetter(rlzs_by_gsim, block, self.sitecol, imts,
                                min_iml, oq.truncation_level, correl_model,
                                samples)
             yield getter, oq, monitor
Example #2
0
 def gen_args(self, ebruptures):
     """
     :param ebruptures: a list of EBRupture objects to be split
     :yields: the arguments for compute_gmfs_and_curves
     """
     oq = self.oqparam
     monitor = self.monitor(self.core_task.__name__)
     monitor.oqparam = oq
     imts = list(oq.imtls)
     min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
     grp_trt = {sg.id: sg.trt for sm in self.csm.info.source_models
                for sg in sm.src_groups}
     rlzs_by_grp = self.rlzs_assoc.get_rlzs_by_grp_id()
     correl_model = oq.get_correl_model()
     for block in split_in_blocks(
             ebruptures, oq.concurrent_tasks or 1,
             key=operator.attrgetter('grp_id')):
         grp_id = block[0].grp_id
         trt = grp_trt[grp_id]
         gsims = [dic[trt] for dic in self.rlzs_assoc.gsim_by_trt]
         samples = self.rlzs_assoc.samples[grp_id]
         getter = GmfGetter(gsims, block, self.sitecol,
                            imts, min_iml, oq.truncation_level,
                            correl_model, samples)
         yield getter, rlzs_by_grp[grp_id], monitor
Example #3
0
 def execute(self):
     """
     Run in parallel `core_task(sources, sitecol, monitor)`, by
     parallelizing on the ruptures according to their weight and
     tectonic region type.
     """
     oq = self.oqparam
     if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
         return
     monitor = self.monitor(self.core_task.__name__)
     monitor.oqparam = oq
     min_iml = calc.fix_minimum_intensity(
         oq.minimum_intensity, oq.imtls)
     acc = parallel.apply_reduce(
         self.core_task.__func__,
         (self.sesruptures, self.sitecol, oq.imtls, self.rlzs_assoc,
          min_iml, monitor),
         concurrent_tasks=self.oqparam.concurrent_tasks,
         agg=self.combine_curves_and_save_gmfs,
         acc=ProbabilityMap(),
         key=operator.attrgetter('grp_id'),
         weight=operator.attrgetter('weight'))
     if oq.ground_motion_fields:
         self.datastore.set_nbytes('gmf_data')
     return acc
Example #4
0
def _calc_gmfs(dstore, serial, eid):
    oq = dstore['oqparam']
    min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
    csm_info = dstore['csm_info']
    rlzs_assoc = csm_info.get_rlzs_assoc()
    rlzs = rlzs_assoc.realizations
    sitecol = dstore['sitecol'].complete
    N = len(sitecol.complete)
    rup = dstore['sescollection/' + serial]
    correl_model = oq.get_correl_model()
    realizations = rlzs_assoc.get_rlzs_by_grp_id()[rup.grp_id]
    gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F64) for rlz in realizations])
    for sm in csm_info.source_models:
        for sg in sm.src_groups:
            if sg.id == rup.grp_id:
                break
    gsims = [dic[sg.trt] for dic in rlzs_assoc.gsim_by_trt]
    getter = GmfGetter(gsims, [rup], sitecol,
                       oq.imtls, min_iml, oq.truncation_level,
                       correl_model, rlzs_assoc.samples[sg.id])
    array_by_imt = {imt: numpy.zeros(N, gmf_dt) for imt in oq.imtls}
    for rlzname in gmf_dt.names:
        rlz = rlzs[int(rlzname)]
        for sid, gmvdict in zip(getter.sids, getter(rlz)):
            if len(gmvdict):
                for imt in oq.imtls:
                    array_by_imt[imt][rlzname][sid] = gmvdict[imt]['gmv']
    return sorted(array_by_imt.items())
Example #5
0
 def gen_args(self, ruptures_by_grp):
     """
     :param ruptures_by_grp: a dictionary of EBRupture objects
     :yields: the arguments for compute_gmfs_and_curves
     """
     oq = self.oqparam
     monitor = self.monitor(self.core_task.__name__)
     monitor.oqparam = oq
     imts = list(oq.imtls)
     min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
     self.grp_trt = self.csm.info.grp_trt()
     rlzs_by_grp = self.rlzs_assoc.get_rlzs_by_grp_id()
     correl_model = oq.get_correl_model()
     for grp_id in ruptures_by_grp:
         ruptures = ruptures_by_grp[grp_id]
         if not ruptures:
             continue
         for block in split_in_blocks(ruptures, oq.concurrent_tasks or 1):
             trt = self.grp_trt[grp_id]
             gsims = [dic[trt] for dic in self.rlzs_assoc.gsim_by_trt]
             samples = self.rlzs_assoc.samples[grp_id]
             getter = GmfGetter(gsims, block, self.sitecol,
                                imts, min_iml, oq.truncation_level,
                                correl_model, samples)
             yield getter, rlzs_by_grp[grp_id], monitor
Example #6
0
 def gen_args(self):
     """
     Yield the arguments required by build_starmap, i.e. the
     source models, the asset collection, the riskmodel and others.
     """
     oq = self.oqparam
     correl_model = oq.get_correl_model()
     if not oq.minimum_intensity:
         # infer it from the risk models if not directly set in job.ini
         oq.minimum_intensity = self.riskmodel.get_min_iml()
     min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
     if min_iml.sum() == 0:
         logging.warn('The GMFs are not filtered: '
                      'you may want to set a minimum_intensity')
     else:
         logging.info('minimum_intensity=%s', oq.minimum_intensity)
     self.csm.init_serials()
     imts = list(oq.imtls)
     for sm_id in range(len(self.csm.source_models)):
         ssm = self.csm.get_model(sm_id)
         monitor = self.monitor.new(
             avg_losses=oq.avg_losses,
             ses_per_logic_tree_path=oq.ses_per_logic_tree_path,
             maximum_distance=oq.maximum_distance,
             samples=ssm.source_models[0].samples,
             seed=ssm.source_model_lt.seed)
         yield (ssm, self.sitecol, self.assetcol, self.riskmodel,
                imts, oq.truncation_level, correl_model, min_iml, monitor)
Example #7
0
 def init(self):
     """
     Set the random seed passed to the SourceManager and the
     minimum_intensity dictionary.
     """
     oq = self.oqparam
     self.random_seed = oq.random_seed
     self.rlzs_assoc = self.datastore['csm_info'].get_rlzs_assoc()
     self.min_iml = calc.fix_minimum_intensity(
         oq.minimum_intensity, oq.imtls)
Example #8
0
 def init(self):
     """
     Set the random seed passed to the SourceManager and the
     minimum_intensity dictionary.
     """
     oq = self.oqparam
     self.min_iml = calc.fix_minimum_intensity(oq.minimum_intensity,
                                               oq.imtls)
     self.rupser = calc.RuptureSerializer(self.datastore)
     self.csm_info = self.datastore['csm_info']
Example #9
0
 def init(self):
     """
     Set the random seed passed to the SourceManager and the
     minimum_intensity dictionary.
     """
     oq = self.oqparam
     self.random_seed = oq.random_seed
     self.rlzs_assoc = self.datastore['csm_info'].get_rlzs_assoc()
     self.min_iml = calc.fix_minimum_intensity(oq.minimum_intensity,
                                               oq.imtls)
     self.rup_data = {}
Example #10
0
 def get_min_iml(self, oq):
     # set the minimum_intensity
     if hasattr(self, 'riskmodel') and not oq.minimum_intensity:
         # infer it from the risk models if not directly set in job.ini
         oq.minimum_intensity = self.riskmodel.get_min_iml()
     min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
     if min_iml.sum() == 0:
         logging.warn('The GMFs are not filtered: '
                      'you may want to set a minimum_intensity')
     else:
         logging.info('minimum_intensity=%s', oq.minimum_intensity)
     return min_iml
Example #11
0
def export_gmf_scenario_csv(ekey, dstore):
    what = ekey[0].split('/')
    if len(what) == 1:
        raise ValueError('Missing "/rup-\d+"')
    oq = dstore['oqparam']
    csm_info = dstore['csm_info']
    rlzs_assoc = csm_info.get_rlzs_assoc()
    samples = csm_info.get_samples_by_grp()
    imts = list(oq.imtls)
    mo = re.match('rup-(\d+)$', what[1])
    if mo is None:
        raise ValueError("Invalid format: %r does not match 'rup-(\d+)$'" %
                         what[1])
    rup_id = int(mo.group(1))
    grp_ids = sorted(int(grp[4:]) for grp in dstore['ruptures'])
    events = dstore['events']
    ruptures = list(calc._get_ruptures(dstore, events, grp_ids, rup_id))
    if not ruptures:
        logging.warn('There is no rupture %d', rup_id)
        return []
    [ebr] = ruptures
    rlzs_by_gsim = rlzs_assoc.get_rlzs_by_gsim(ebr.grp_id)
    samples = samples[ebr.grp_id]
    min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
    correl_model = oq.get_correl_model()
    sitecol = dstore['sitecol'].complete
    getter = GmfGetter(rlzs_by_gsim, ruptures, sitecol, imts, min_iml,
                       oq.truncation_level, correl_model, samples)
    getter.init()
    hazardr = getter.get_hazard()
    rlzs = rlzs_assoc.realizations
    fields = ['eid-%03d' % eid for eid in getter.eids]
    dt = numpy.dtype([(f, F32) for f in fields])
    mesh = numpy.zeros(len(ebr.sids), [('lon', F64), ('lat', F64)])
    mesh['lon'] = sitecol.lons[ebr.sids]
    mesh['lat'] = sitecol.lats[ebr.sids]
    writer = writers.CsvWriter(fmt='%.5f')
    for rlzi in range(len(rlzs)):
        hazard = hazardr[rlzi]
        for imti, imt in enumerate(imts):
            gmfs = numpy.zeros(len(ebr.sids), dt)
            for s, sid in enumerate(ebr.sids):
                for rec in hazard[sid]:
                    event = 'eid-%03d' % rec['eid']
                    gmfs[s][event] = rec['gmv'][imti]
            dest = dstore.build_fname('gmf',
                                      'rup-%s-rlz-%s-%s' % (rup_id, rlzi, imt),
                                      'csv')
            data = util.compose_arrays(mesh, gmfs)
            writer.save(data, dest)
    return writer.getsaved()
Example #12
0
def export_gmf_scenario_csv(ekey, dstore):
    what = ekey[0].split('/')
    if len(what) == 1:
        raise ValueError('Missing "/rup-\d+"')
    oq = dstore['oqparam']
    csm_info = dstore['csm_info']
    rlzs_assoc = csm_info.get_rlzs_assoc()
    samples = csm_info.get_samples_by_grp()
    num_ruptures = len(dstore['ruptures'])
    imts = list(oq.imtls)
    mo = re.match('rup-(\d+)$', what[1])
    if mo is None:
        raise ValueError("Invalid format: %r does not match 'rup-(\d+)$'" %
                         what[1])
    ridx = int(mo.group(1))
    assert 0 <= ridx < num_ruptures, ridx
    ruptures = list(RuptureGetter(dstore, slice(ridx, ridx + 1)))
    [ebr] = ruptures
    rlzs_by_gsim = rlzs_assoc.get_rlzs_by_gsim(ebr.grp_id)
    samples = samples[ebr.grp_id]
    min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
    sitecol = dstore['sitecol'].complete
    getter = GmfGetter(rlzs_by_gsim, ruptures, sitecol, oq, min_iml, samples)
    getter.init()
    sids = getter.computers[0].sids
    hazardr = getter.get_hazard()
    rlzs = rlzs_assoc.realizations
    fields = ['eid-%03d' % eid for eid in getter.eids]
    dt = numpy.dtype([(f, F32) for f in fields])
    mesh = numpy.zeros(len(sids), [('lon', F64), ('lat', F64)])
    mesh['lon'] = sitecol.lons[sids]
    mesh['lat'] = sitecol.lats[sids]
    writer = writers.CsvWriter(fmt='%.5f')
    for rlzi in range(len(rlzs)):
        hazard = hazardr[rlzi]
        for imti, imt in enumerate(imts):
            gmfs = numpy.zeros(len(sids), dt)
            for s, sid in enumerate(sids):
                for rec in hazard[sid]:
                    event = 'eid-%03d' % rec['eid']
                    gmfs[s][event] = rec['gmv'][imti]
            dest = dstore.build_fname(
                'gmf', 'rup-%s-rlz-%s-%s' % (ebr.serial, rlzi, imt), 'csv')
            data = util.compose_arrays(mesh, gmfs)
            writer.save(data, dest)
    return writer.getsaved()
Example #13
0
def _get_gmfs(dstore, serial, eid):
    oq = dstore['oqparam']
    min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
    rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
    sitecol = dstore['sitecol'].complete
    N = len(sitecol.complete)
    rup = dstore['sescollection/' + serial]
    correl_model = readinput.get_correl_model(oq)
    gsims = rlzs_assoc.gsims_by_trt_id[rup.trt_id]
    rlzs = [
        rlz for gsim in map(str, gsims) for rlz in rlzs_assoc[rup.trt_id, gsim]
    ]
    gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F32) for rlz in rlzs])
    gmfadict = create(calc.GmfColl, [rup], sitecol, oq.imtls, rlzs_assoc,
                      oq.truncation_level, correl_model, min_iml).by_rlzi()
    for imti, imt in enumerate(oq.imtls):
        gmfa = numpy.zeros(N, gmf_dt)
        for rlzname in gmf_dt.names:
            rlzi = int(rlzname)
            gmvs = get_array(gmfadict[rlzi], eid=eid, imti=imti)['gmv']
            gmfa[rlzname][rup.indices] = gmvs
        yield gmfa, imt
Example #14
0
def _get_gmfs(dstore, serial, eid):
    oq = dstore['oqparam']
    min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
    rlzs_assoc = dstore['csm_info'].get_rlzs_assoc()
    sitecol = dstore['sitecol'].complete
    N = len(sitecol.complete)
    rup = dstore['sescollection/' + serial]
    correl_model = readinput.get_correl_model(oq)
    gsims = rlzs_assoc.gsims_by_grp_id[rup.grp_id]
    rlzs = [rlz for gsim in map(str, gsims)
            for rlz in rlzs_assoc[rup.grp_id, gsim]]
    gmf_dt = numpy.dtype([('%03d' % rlz.ordinal, F32) for rlz in rlzs])
    gmfadict = create(calc.GmfColl,
                      [rup], sitecol, oq.imtls, rlzs_assoc,
                      oq.truncation_level, correl_model, min_iml).by_rlzi()
    for imti, imt in enumerate(oq.imtls):
        gmfa = numpy.zeros(N, gmf_dt)
        for rlzname in gmf_dt.names:
            rlzi = int(rlzname)
            gmvs = get_array(gmfadict[rlzi], eid=eid, imti=imti)['gmv']
            gmfa[rlzname][rup.indices] = gmvs
        yield gmfa, imt
Example #15
0
 def execute(self):
     """
     Run in parallel `core_task(sources, sitecol, monitor)`, by
     parallelizing on the ruptures according to their weight and
     tectonic region type.
     """
     oq = self.oqparam
     if not oq.hazard_curves_from_gmfs and not oq.ground_motion_fields:
         return
     monitor = self.monitor(self.core_task.__name__)
     monitor.oqparam = oq
     min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
     acc = parallel.apply_reduce(
         self.core_task.__func__, (self.sesruptures, self.sitecol, oq.imtls,
                                   self.rlzs_assoc, min_iml, monitor),
         concurrent_tasks=self.oqparam.concurrent_tasks,
         agg=self.combine_curves_and_save_gmfs,
         acc=ProbabilityMap(),
         key=operator.attrgetter('trt_id'),
         weight=operator.attrgetter('weight'))
     if oq.ground_motion_fields:
         self.datastore.set_nbytes('gmf_data')
     return acc
Example #16
0
    def execute(self):
        """
        Run the event_based_risk calculator and aggregate the results
        """
        oq = self.oqparam
        correl_model = oq.get_correl_model()
        self.N = len(self.assetcol)
        self.E = sum(len(v) for v in self.datastore['events'].values())
        logging.info('Populating the risk inputs')
        all_ruptures = []
        preprecalc = getattr(self.precalc, 'precalc', None)
        if preprecalc:  # the ruptures are already in memory
            for grp_id, sesruptures in preprecalc.result.items():
                for sr in sesruptures:
                    all_ruptures.append(sr)
        else:  # read the ruptures from the datastore
            for serial in self.datastore['sescollection']:
                rup = self.datastore['sescollection/' + serial]
                all_ruptures.append(rup)
        all_ruptures.sort(key=operator.attrgetter('serial'))
        if not self.riskmodel.covs:
            # do not generate epsilons
            eps = None
        else:
            eps = riskinput.make_eps(
                self.assets_by_site, self.E, oq.master_seed,
                oq.asset_correlation)
            logging.info('Generated %s epsilons', eps.shape)

        # preparing empty datasets
        loss_types = self.riskmodel.loss_types
        self.C = self.oqparam.loss_curve_resolution
        self.L = L = len(loss_types)
        self.R = R = len(self.rlzs_assoc.realizations)
        self.I = self.oqparam.insured_losses

        # ugly: attaching attributes needed in the task function
        mon = self.monitor
        mon.num_assets = self.count_assets()
        mon.avg_losses = self.oqparam.avg_losses
        mon.asset_loss_table = self.oqparam.asset_loss_table
        mon.insured_losses = self.I
        mon.ses_ratio = (
            oq.risk_investigation_time or oq.investigation_time) / (
                oq.investigation_time * oq.ses_per_logic_tree_path)

        self.N = N = len(self.assetcol)
        self.E = sum(len(v) for v in self.datastore['events'].values())

        # average losses, stored in a composite array of shape N, R
        self.avg_losses = numpy.zeros((N, R), oq.loss_dt())

        self.ass_loss_table = square(L, R, lambda: None)
        self.agg_loss_table = square(L, R, lambda: None)

        self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes(
            self.I)
        for (l, r) in itertools.product(range(L), range(R)):
            lt = loss_types[l]
            if self.oqparam.asset_loss_table:
                self.ass_loss_table[l, r] = self.datastore.create_dset(
                    'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt)
            self.agg_loss_table[l, r] = self.datastore.create_dset(
                'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt)

        self.saved = collections.Counter()  # nbytes per HDF5 key
        self.ass_bytes = 0
        self.agg_bytes = 0
        self.gmfbytes = 0
        rlz_ids = getattr(self.oqparam, 'rlz_ids', ())
        if rlz_ids:
            self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids)

        if not oq.minimum_intensity:
            # infer it from the risk models if not directly set in job.ini
            oq.minimum_intensity = self.riskmodel.get_min_iml()
        min_iml = calc.fix_minimum_intensity(
            oq.minimum_intensity, oq.imtls)
        if min_iml.sum() == 0:
            logging.warn('The GMFs are not filtered: '
                         'you may want to set a minimum_intensity')
        else:
            logging.info('minimum_intensity=%s', oq.minimum_intensity)
        csm_info = self.datastore['csm_info']
        grp_trt = {sg.id: sg.trt for sm in csm_info.source_models
                   for sg in sm.src_groups}
        with self.monitor('building riskinputs', autoflush=True):
            riskinputs = self.riskmodel.build_inputs_from_ruptures(
                grp_trt, list(oq.imtls), self.sitecol.complete, all_ruptures,
                oq.truncation_level, correl_model, min_iml, eps,
                oq.concurrent_tasks or 1)
            # NB: I am using generators so that the tasks are submitted one at
            # the time, without keeping all of the arguments in memory
            res = starmap(
                self.core_task.__func__,
                ((riskinput, self.riskmodel, self.rlzs_assoc,
                  self.assetcol, self.monitor.new('task'))
                 for riskinput in riskinputs)).submit_all()
        acc = functools.reduce(self.agg, res, AccumDict())
        self.save_data_transfer(res)
        return acc
Example #17
0
def compute_ruptures_gmfs_curves(
        source_models, sitecol, rlzs_assoc, monitor):
    """
    Returns the ruptures as a TRT set
    :param source_models:
        A list of UCERF source models, one per branch
    :param sitecol:
        Site collection :class:`openquake.hazardlib.site.SiteCollection`
    :param rlzs_assoc:
        Instance of :class:`openquake.commonlib.source.RlzsAssoc`
    :param monitor:
        Instance of :class:`openquake.baselib.performance.Monitor`
    :returns:
        Dictionary of rupture instances associated to a TRT ID
    """
    oq = monitor.oqparam
    correl_model = oq.get_correl_model()
    imts = list(oq.imtls)
    min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, imts)
    integration_distance = oq.maximum_distance[DEFAULT_TRT]
    res = AccumDict()
    res.calc_times = AccumDict()
    serial = 1
    event_mon = monitor('sampling ruptures', measuremem=False)
    res['ruptures'] = rupdic = AccumDict()
    rupdic.num_events = 0
    rupdic.trt = DEFAULT_TRT
    rlzs_by_grp = rlzs_assoc.get_rlzs_by_grp_id()
    for grp_id, source_model in enumerate(source_models):
        [grp] = source_model.src_groups  # one source group per source model
        [ucerf] = grp  # one source per source group
        t0 = time.time()
        # set the seed before calling generate_event_set
        numpy.random.seed(oq.random_seed + grp_id)
        ses_ruptures = []
        eid = 0
        ucerf.idx_set = ucerf.build_idx_set()
        background_sids = ucerf.get_background_sids(
            sitecol, integration_distance)
        for ses_idx in range(1, oq.ses_per_logic_tree_path + 1):
            with event_mon:
                rups, n_occs = ucerf.generate_event_set(background_sids)
            for i, rup in enumerate(rups):
                rup.seed = oq.random_seed  # to think
                rrup = rup.surface.get_min_distance(sitecol.mesh)
                r_sites = sitecol.filter(rrup <= integration_distance)
                if r_sites is None:
                    continue
                indices = r_sites.indices
                events = []
                for j in range(n_occs[i]):
                    events.append((eid, ses_idx, j, 0))  # 0 is the sampling ID
                    eid += 1
                if events:
                    ses_ruptures.append(
                        event_based.EBRupture(
                            rup, indices,
                            numpy.array(events, event_based.event_dt),
                            ucerf.source_id, grp_id, serial))
                    serial += 1
                    rupdic.num_events += len(events)
        res['ruptures'][grp_id] = ses_ruptures
        gsims = [dic[DEFAULT_TRT] for dic in rlzs_assoc.gsim_by_trt]
        gg = riskinput.GmfGetter(gsims, ses_ruptures, sitecol,
                                 imts, min_iml, oq.truncation_level,
                                 correl_model, rlzs_assoc.samples[grp_id])
        rlzs = rlzs_by_grp[grp_id]
        res.update(event_based.compute_gmfs_and_curves(gg, rlzs, monitor))
        res.calc_times[grp_id] = (ucerf.source_id, len(sitecol),
                                  time.time() - t0)
    return res
Example #18
0
    def execute(self):
        """
        Run the event_based_risk calculator and aggregate the results
        """
        oq = self.oqparam
        correl_model = readinput.get_correl_model(oq)
        self.N = len(self.assetcol)
        self.E = len(self.etags)
        logging.info('Populating the risk inputs')
        rlzs_by_tr_id = self.rlzs_assoc.get_rlzs_by_trt_id()
        num_rlzs = {t: len(rlzs) for t, rlzs in rlzs_by_tr_id.items()}
        num_assets = {
            sid: len(self.assets_by_site[sid])
            for sid in self.sitecol.sids
        }
        all_ruptures = []
        for serial in self.datastore['sescollection']:
            rup = self.datastore['sescollection/' + serial]
            rup.set_weight(num_rlzs, num_assets)
            all_ruptures.append(rup)
        all_ruptures.sort(key=operator.attrgetter('serial'))
        if not self.riskmodel.covs:
            # do not generate epsilons
            eps = None
        else:
            eps = riskinput.make_eps(self.assets_by_site, self.E,
                                     oq.master_seed, oq.asset_correlation)
            logging.info('Generated %s epsilons', eps.shape)

        # preparing empty datasets
        loss_types = self.riskmodel.loss_types
        self.C = self.oqparam.loss_curve_resolution
        self.L = L = len(loss_types)
        self.R = R = len(self.rlzs_assoc.realizations)
        self.I = self.oqparam.insured_losses

        # ugly: attaching attributes needed in the task function
        mon = self.monitor
        mon.num_assets = self.count_assets()
        mon.avg_losses = self.oqparam.avg_losses
        mon.asset_loss_table = self.oqparam.asset_loss_table
        mon.insured_losses = self.I
        mon.ses_ratio = (oq.risk_investigation_time or
                         oq.investigation_time) / (oq.investigation_time *
                                                   oq.ses_per_logic_tree_path)

        self.N = N = len(self.assetcol)
        self.E = len(self.datastore['etags'])

        # average losses, stored in a composite array of shape N, R
        multi_avg_dt = self.riskmodel.loss_type_dt(insured=self.I)
        self.avg_losses = numpy.zeros((N, R), multi_avg_dt)

        self.ass_loss_table = square(L, R, lambda: None)
        self.agg_loss_table = square(L, R, lambda: None)

        self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes(
            self.I)
        for (l, r) in itertools.product(range(L), range(R)):
            lt = loss_types[l]
            if self.oqparam.asset_loss_table:
                self.ass_loss_table[l, r] = self.datastore.create_dset(
                    'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt)
            self.agg_loss_table[l, r] = self.datastore.create_dset(
                'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt)

        self.saved = collections.Counter()  # nbytes per HDF5 key
        self.ass_bytes = 0
        self.agg_bytes = 0
        self.gmfbytes = 0
        rlz_ids = getattr(self.oqparam, 'rlz_ids', ())
        if rlz_ids:
            self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids)

        if not oq.minimum_intensity:
            # infer it from the risk models if not directly set in job.ini
            oq.minimum_intensity = self.riskmodel.get_min_iml()
        min_iml = calc.fix_minimum_intensity(oq.minimum_intensity, oq.imtls)
        if min_iml.sum() == 0:
            logging.warn('The GMFs are not filtered: '
                         'you may want to set a minimum_intensity')
        else:
            logging.info('minimum_intensity=%s', oq.minimum_intensity)

        with self.monitor('building riskinputs', autoflush=True):
            riskinputs = self.riskmodel.build_inputs_from_ruptures(
                self.sitecol.complete, all_ruptures, oq.truncation_level,
                correl_model, min_iml, eps, oq.concurrent_tasks or 1)
            # NB: I am using generators so that the tasks are submitted one at
            # the time, without keeping all of the arguments in memory
            tm = starmap(self.core_task.__func__,
                         ((riskinput, self.riskmodel, self.rlzs_assoc,
                           self.assetcol, self.monitor.new('task'))
                          for riskinput in riskinputs))
        res = tm.reduce(agg=self.agg)
        self.save_data_transfer(tm)
        return res
Example #19
0
    def execute(self):
        """
        Run the event_based_risk calculator and aggregate the results
        """
        oq = self.oqparam
        correl_model = readinput.get_correl_model(oq)
        self.N = len(self.assetcol)
        self.E = len(self.etags)
        logging.info('Populating the risk inputs')
        rlzs_by_tr_id = self.rlzs_assoc.get_rlzs_by_trt_id()
        num_rlzs = {t: len(rlzs) for t, rlzs in rlzs_by_tr_id.items()}
        num_assets = {sid: len(self.assets_by_site[sid])
                      for sid in self.sitecol.sids}
        all_ruptures = []
        for serial in self.datastore['sescollection']:
            rup = self.datastore['sescollection/' + serial]
            rup.set_weight(num_rlzs, num_assets)
            all_ruptures.append(rup)
        all_ruptures.sort(key=operator.attrgetter('serial'))
        if not self.riskmodel.covs:
            # do not generate epsilons
            eps = None
        else:
            eps = riskinput.make_eps(
                self.assets_by_site, self.E, oq.master_seed,
                oq.asset_correlation)
            logging.info('Generated %s epsilons', eps.shape)

        # preparing empty datasets
        loss_types = self.riskmodel.loss_types
        self.C = self.oqparam.loss_curve_resolution
        self.L = L = len(loss_types)
        self.R = R = len(self.rlzs_assoc.realizations)
        self.I = self.oqparam.insured_losses

        # ugly: attaching attributes needed in the task function
        mon = self.monitor
        mon.num_assets = self.count_assets()
        mon.avg_losses = self.oqparam.avg_losses
        mon.asset_loss_table = self.oqparam.asset_loss_table
        mon.insured_losses = self.I
        mon.ses_ratio = (
            oq.risk_investigation_time or oq.investigation_time) / (
                oq.investigation_time * oq.ses_per_logic_tree_path)

        self.N = N = len(self.assetcol)
        self.E = len(self.datastore['etags'])

        # average losses, stored in a composite array of shape N, R
        multi_avg_dt = self.riskmodel.loss_type_dt(insured=self.I)
        self.avg_losses = numpy.zeros((N, R), multi_avg_dt)

        self.ass_loss_table = square(L, R, lambda: None)
        self.agg_loss_table = square(L, R, lambda: None)

        self.ela_dt, self.elt_dt = mon.ela_dt, mon.elt_dt = build_el_dtypes(
            self.I)
        for (l, r) in itertools.product(range(L), range(R)):
            lt = loss_types[l]
            if self.oqparam.asset_loss_table:
                self.ass_loss_table[l, r] = self.datastore.create_dset(
                    'ass_loss_table/rlz-%03d/%s' % (r, lt), self.ela_dt)
            self.agg_loss_table[l, r] = self.datastore.create_dset(
                'agg_loss_table/rlz-%03d/%s' % (r, lt), self.elt_dt)

        self.saved = collections.Counter()  # nbytes per HDF5 key
        self.ass_bytes = 0
        self.agg_bytes = 0
        self.gmfbytes = 0
        rlz_ids = getattr(self.oqparam, 'rlz_ids', ())
        if rlz_ids:
            self.rlzs_assoc = self.rlzs_assoc.extract(rlz_ids)

        if not oq.minimum_intensity:
            # infer it from the risk models if not directly set in job.ini
            oq.minimum_intensity = self.riskmodel.get_min_iml()
        min_iml = calc.fix_minimum_intensity(
            oq.minimum_intensity, oq.imtls)
        if min_iml.sum() == 0:
            logging.warn('The GMFs are not filtered: '
                         'you may want to set a minimum_intensity')
        else:
            logging.info('minimum_intensity=%s', oq.minimum_intensity)

        with self.monitor('building riskinputs', autoflush=True):
            riskinputs = self.riskmodel.build_inputs_from_ruptures(
                self.sitecol.complete, all_ruptures, oq.truncation_level,
                correl_model, min_iml, eps, oq.concurrent_tasks or 1)
            # NB: I am using generators so that the tasks are submitted one at
            # the time, without keeping all of the arguments in memory
            tm = starmap(
                self.core_task.__func__,
                ((riskinput, self.riskmodel, self.rlzs_assoc,
                  self.assetcol, self.monitor.new('task'))
                 for riskinput in riskinputs))
        return tm.reduce(agg=self.agg, posthook=self.save_data_transfer)