Ejemplo n.º 1
0
def pmap_from_grp(
        sources, sites, imtls, gsims, truncation_level=None,
        source_site_filter='SourceSitesFilter', maximum_distance=None, bbs=(),
        monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if source_site_filter == 'SourceSitesFilter':  # default
        source_site_filter = (
            filters.SourceSitesFilter(maximum_distance)
            if maximum_distance else filters.source_site_noop_filter)
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maximum_distance)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mon = monitor('computing poes', measuremem=False)
        disagg_mon = monitor('get closest points', measuremem=False)
        monitor.calc_times = []  # pairs (src_id, delta_t)
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        for src, s_sites in source_site_filter(sources, sites):
            t0 = time.time()
            pmap |= poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs,
                            ctx_mon, pne_mon, disagg_mon)
            # we are attaching the calculation times to the monitor
            # so that oq-lite (and the engine) can store them
            monitor.calc_times.append((src.id, time.time() - t0))
            # NB: source.id is an integer; it should not be confused
            # with source.source_id, which is a string
        monitor.eff_ruptures = pne_mon.counts  # contributing ruptures
        return pmap
Ejemplo n.º 2
0
    def wrapped(*args):
        """
        Initialize logs, make sure the job is still running, and run the task
        code surrounded by a try-except. If any error occurs, log it as a
        critical failure.
        """
        # the last argument is assumed to be a monitor
        monitor = args[-1]
        job = models.OqJob.objects.get(id=monitor.job_id)
        if job.is_running is False:
            # the job was killed, it is useless to run the task
            raise JobNotRunning(monitor.job_id)

        # it is important to save the task id soon, so that
        # the revoke functionality can work
        with monitor("storing task id", task=tsk, autoflush=True):
            pass

        with logs.handle(job):
            check_mem_usage()  # warn if too much memory is used
            # run the task
            try:
                total = "total " + task_func.__name__
                with monitor(total, task=tsk):
                    with GroundShakingIntensityModel.forbid_instantiation():
                        return task_func(*args)
            finally:
                # save on the db
                CacheInserter.flushall()
                # the task finished, we can remove from the performance
                # table the associated row 'storing task id'
                models.Performance.objects.filter(
                    oq_job=job, operation="storing task id", task_id=tsk.request.id
                ).delete()
Ejemplo n.º 3
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    if getattr(group, 'src_interdep', None) == 'mutex':
        mutex_weight = {
            src.source_id: weight
            for src, weight in zip(group.sources, group.srcs_weights)
        }
    else:
        mutex_weight = None
    grp_ids = set()
    for src in group:
        grp_ids.update(src.src_group_ids)
    maxdist = src_filter.integration_distance
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('make_contexts', measuremem=False)
        poe_mon = monitor('get_poes', measuremem=False)
        pmap = AccumDict({
            grp_id: ProbabilityMap(len(imtls.array), len(gsims))
            for grp_id in grp_ids
        })
        # AccumDict of arrays with 4 elements weight, nsites, calc_time, split
        pmap.calc_times = AccumDict(accum=numpy.zeros(4))
        pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
        for src, s_sites in src_filter(group):  # filter now
            t0 = time.time()
            indep = group.rup_interdep == 'indep' if mutex_weight else True
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
                                    poe_mon, indep)
            if mutex_weight:  # mutex sources
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap[group.id].setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            elif poemap:
                for grp_id in src.src_group_ids:
                    pmap[grp_id] |= poemap
            src_id = src.source_id.split(':', 1)[0]
            pmap.calc_times[src_id] += numpy.array(
                [src.weight, len(s_sites),
                 time.time() - t0, 1])
            # storing the number of contributing ruptures too
            pmap.eff_ruptures += {
                grp_id: getattr(poemap, 'eff_ruptures', 0)
                for grp_id in src.src_group_ids
            }
        if mutex_weight and group.grp_probability is not None:
            pmap[group.id] *= group.grp_probability
        return pmap
Ejemplo n.º 4
0
 def wrapper(*args):
     check_mem_usage()  # check if too much memory is used
     monitor = args[-1]
     monitor.flush = NoFlush(monitor, func.__name__)
     with monitor('total ' + func.__name__, measuremem=True), \
             GroundShakingIntensityModel.forbid_instantiation():
         result = func(*args)
     rec_delattr(monitor, 'flush')
     return result
Ejemplo n.º 5
0
 def wrapper(*args):
     monitor = args[-1]
     check_mem_usage(monitor)  # check if too much memory is used
     monitor.flush = NoFlush(monitor, func.__name__)
     with monitor('total ' + func.__name__, measuremem=True), \
             GroundShakingIntensityModel.forbid_instantiation():
         result = func(*args)
     rec_delattr(monitor, 'flush')
     return result
Ejemplo n.º 6
0
 def wrapper(*args):
     monitor = args[-1]
     check_mem_usage(monitor)  # check if too much memory is used
     monitor.flush = NoFlush(monitor, func.__name__)
     with monitor('total ' + func.__name__, measuremem=True), \
             GroundShakingIntensityModel.forbid_instantiation():
         result = func(*args)
     # NB: flush must not be called in the workers - they must not
     # have access to the datastore - so we remove it
     rec_delattr(monitor, 'flush')
     return result
Ejemplo n.º 7
0
def pmap_from_grp(sources,
                  source_site_filter,
                  imtls,
                  gsims,
                  truncation_level=None,
                  bbs=(),
                  monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.sources
        trt = sources[0].tectonic_region_type
    else:  # list of sources
        trt = sources[0].tectonic_region_type
        group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
    try:
        maxdist = source_site_filter.integration_distance[trt]
    except:
        maxdist = source_site_filter.integration_distance
    if hasattr(gsims, 'keys'):  # dictionary trt -> gsim
        gsims = [gsims[trt]]
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mon = monitor('computing poes', measuremem=False)
        disagg_mon = monitor('get closest points', measuremem=False)
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = sources[0].src_group_id
        for src, s_sites in source_site_filter(sources):
            t0 = time.time()
            poemap = poe_map(src, s_sites, imtls, cmaker, truncation_level,
                             bbs, group.rup_interdep == 'indep', ctx_mon,
                             pne_mon, disagg_mon)
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = float(group.srcs_weights[src.source_id])
                for sid in poemap:
                    pmap[sid] += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts}
        return pmap
Ejemplo n.º 8
0
def pmap_from_grp(
        sources, source_site_filter, imtls, gsims, truncation_level=None,
        bbs=(), monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.src_list
    else:
        group = SourceGroup(sources, 'src_group', 'indep', 'indep')
        sources = group.src_list
    trt = sources[0].tectonic_region_type
    try:
        maxdist = source_site_filter.integration_distance[trt]
    except:
        maxdist = source_site_filter.integration_distance
    if hasattr(gsims, 'keys'):
        gsims = [gsims[trt]]
    # check all the sources belong to the same tectonic region
    trts = set(src.tectonic_region_type for src in sources)
    assert len(trts) == 1, 'Multiple TRTs: %s' % ', '.join(trts)

    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mon = monitor('computing poes', measuremem=False)
        disagg_mon = monitor('get closest points', measuremem=False)
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = sources[0].src_group_id
        for src, s_sites in source_site_filter(sources):
            t0 = time.time()
            poemap = poe_map(
                src, s_sites, imtls, cmaker, truncation_level, bbs,
                group.rup_interdep == 'indep', ctx_mon, pne_mon, disagg_mon)
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = float(group.srcs_weights[src.source_id])
                for sid in poemap:
                    pmap[sid] += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts}
        return pmap
Ejemplo n.º 9
0
 def wrapper(*args):
     monitor = args[-1]
     check_mem_usage(monitor)  # check if too much memory is used
     monitor.flush = NoFlush(monitor, func.__name__)
     with monitor('total ' + func.__name__, measuremem=True), \
             GroundShakingIntensityModel.forbid_instantiation():
         result = func(*args)
     # NB: flush must not be called in the workers - they must not
     # have access to the datastore - so we remove it
     rec_delattr(monitor, 'flush')
     return result
Ejemplo n.º 10
0
def pmap_from_trt(sources, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    maxdist = src_filter.integration_distance
    srcs = []
    grp_ids = set()
    for src in sources:
        if hasattr(src, '__iter__'):  # MultiPointSource
            srcs.extend(src)
        else:
            srcs.append(src)
        grp_ids.update(src.src_group_ids)
    del sources
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mons = [
            monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims
        ]
        pmap = AccumDict({
            grp_id: ProbabilityMap(len(imtls.array), len(gsims))
            for grp_id in grp_ids
        })
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
        for src, s_sites in src_filter(srcs):
            t0 = time.time()
            poe = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon,
                          pne_mons)
            for grp_id in src.src_group_ids:
                pmap[grp_id] |= poe
            pmap.calc_times.append(
                (src.source_id, src.weight, len(s_sites), time.time() - t0))
            # storing the number of contributing ruptures too
            pmap.eff_ruptures += {
                grp_id: poe.eff_ruptures
                for grp_id in src.src_group_ids
            }
        return pmap
Ejemplo n.º 11
0
def pmap_from_trt(sources, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    grp_ids = set()
    for src in sources:
        grp_ids.update(src.src_group_ids)
    maxdist = src_filter.integration_distance
    srcs = sum([split_source(src) for src in sources], [])  # split first
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('make_contexts', measuremem=False)
        poe_mon = monitor('get_poes', measuremem=False)
        pmap = AccumDict({
            grp_id: ProbabilityMap(len(imtls.array), len(gsims))
            for grp_id in grp_ids
        })
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
        for src, s_sites in src_filter(srcs):  # filter now
            t0 = time.time()
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
                                    poe_mon)
            if poemap:
                for grp_id in src.src_group_ids:
                    pmap[grp_id] |= poemap
            pmap.calc_times.append(
                (src.source_id, src.weight, len(s_sites), time.time() - t0))
            # storing the number of contributing ruptures too
            pmap.eff_ruptures += {
                grp_id: getattr(poemap, 'eff_ruptures', 0)
                for grp_id in src.src_group_ids
            }
        return pmap
Ejemplo n.º 12
0
def pmap_from_grp(sources,
                  sites,
                  imtls,
                  gsims,
                  truncation_level=None,
                  source_site_filter='SourceSitesFilter',
                  maximum_distance=None,
                  bbs=(),
                  monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if source_site_filter == 'SourceSitesFilter':  # default
        source_site_filter = (filters.SourceSitesFilter(maximum_distance)
                              if maximum_distance else
                              filters.source_site_noop_filter)
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maximum_distance)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mon = monitor('computing poes', measuremem=False)
        disagg_mon = monitor('get closest points', measuremem=False)
        monitor.calc_times = []  # pairs (src_id, delta_t)
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        for src, s_sites in source_site_filter(sources, sites):
            t0 = time.time()
            pmap |= poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs,
                            ctx_mon, pne_mon, disagg_mon)
            # we are attaching the calculation times to the monitor
            # so that oq-lite (and the engine) can store them
            monitor.calc_times.append(
                (src.source_id, len(s_sites), time.time() - t0))
            # NB: source.id is an integer; it should not be confused
            # with source.source_id, which is a string
        monitor.eff_ruptures = pne_mon.counts  # contributing ruptures
        return pmap
Ejemplo n.º 13
0
def pmap_from_grp(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a dictionary {grp_id: ProbabilityMap instance}
    """
    mutex_weight = {
        src.source_id: weight
        for src, weight in zip(group.sources, group.srcs_weights)
    }
    maxdist = src_filter.integration_distance
    srcs = sum([split_source(src) for src in group.sources], [])
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('make_contexts', measuremem=False)
        poe_mon = monitor('get_poes', measuremem=False)
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        calc_times = []  # pairs (src_id, delta_t)
        for src, s_sites in src_filter(srcs):
            t0 = time.time()
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
                                    poe_mon, group.rup_interdep == 'indep')
            weight = mutex_weight[src.source_id]
            for sid in poemap:
                pcurve = pmap.setdefault(sid, 0)
                pcurve += poemap[sid] * weight
            calc_times.append(
                (src.source_id, src.weight, len(s_sites), time.time() - t0))
        if group.grp_probability is not None:
            pmap *= group.grp_probability
        acc = AccumDict({group.id: pmap})
        # adding the number of contributing ruptures too
        acc.eff_ruptures = {group.id: ctx_mon.counts}
        acc.calc_times = calc_times
        return acc
Ejemplo n.º 14
0
def pmap_from_grp(sources, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.sources
        trt = sources[0].tectonic_region_type
        mutex_weight = {
            src.source_id: weight
            for src, weight in zip(group.sources, group.srcs_weights)
        }
    else:  # list of sources
        trt = sources[0].tectonic_region_type
        group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
    grp_id = sources[0].src_group_id
    maxdist = src_filter.integration_distance
    if hasattr(gsims, 'keys'):  # dictionary trt -> gsim
        gsims = [gsims[trt]]
    srcs = []
    for src in sources:
        if hasattr(src, '__iter__'):  # MultiPointSource
            srcs.extend(src)
        else:
            srcs.append(src)
    del sources
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mons = [
            monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims
        ]
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = grp_id
        for src, s_sites in src_filter(srcs):
            t0 = time.time()
            poemap = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon,
                             pne_mons, group.rup_interdep == 'indep')
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap.setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, src.weight, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mons[0].counts}
        if group.grp_probability is not None:
            return pmap * group.grp_probability
        return pmap