Ejemplo n.º 1
0
 def __init__(self, oqparam, monitor=Monitor(), calc_id=None):
     self.monitor = monitor
     self.datastore = datastore.DataStore(calc_id)
     self.monitor.calc_id = self.datastore.calc_id
     self.monitor.hdf5path = self.datastore.hdf5path
     self.datastore.export_dir = oqparam.export_dir
     self.oqparam = oqparam
Ejemplo n.º 2
0
 def gen_ctxs(self, ruptures, sites, src_id, mon=Monitor()):
     """
     :param ruptures:
         a list of ruptures generated by the same source
     :param sites:
         a (filtered) SiteCollection
     :param src_id:
         the ID of the source (for debugging purposes)
     :param mon:
         a Monitor object
     :yields:
         fat RuptureContexts
     """
     fewsites = len(sites.complete) <= self.max_sites_disagg
     for rup in ruptures:
         with mon:
             try:
                 ctx, r_sites, dctx = self.make_contexts(
                     getattr(rup, 'sites', sites), rup)
             except FarAwayRupture:
                 continue
             for par in self.REQUIRES_SITES_PARAMETERS:
                 setattr(ctx, par, r_sites[par])
             ctx.sids = r_sites.sids
             ctx.src_id = src_id
             for par in self.REQUIRES_DISTANCES | {'rrup'}:
                 setattr(ctx, par, getattr(dctx, par))
             if fewsites:
                 # get closest point on the surface
                 closest = rup.surface.get_closest_points(sites.complete)
                 ctx.clon = closest.lons[ctx.sids]
                 ctx.clat = closest.lats[ctx.sids]
         yield ctx
Ejemplo n.º 3
0
def grid_point_sources(sources, ps_grid_spacing, monitor=Monitor()):
    """
    :param sources:
        a list of sources with the same grp_id (point sources and not)
    :param ps_grid_spacing:
        value of the point source grid spacing in km; if None, do nothing
    :returns:
        a dict grp_id -> list of non-point sources and collapsed point sources
    """
    grp_id = sources[0].grp_id
    for src in sources[1:]:
        assert src.grp_id == grp_id, (src.grp_id, grp_id)
    if ps_grid_spacing is None:
        return {grp_id: sources}
    out = [src for src in sources if not hasattr(src, 'location')]
    ps = numpy.array([src for src in sources if hasattr(src, 'location')])
    if len(ps) < 2:  # nothing to collapse
        return {grp_id: out + list(ps)}
    coords = _coords(ps)
    deltax = angular_distance(ps_grid_spacing, lat=coords[:, 1].mean())
    deltay = angular_distance(ps_grid_spacing)
    grid = groupby_grid(coords[:, 0], coords[:, 1], deltax, deltay)
    task_no = getattr(monitor, 'task_no', 0)
    for i, idxs in enumerate(grid.values()):
        if len(idxs) > 1:
            cps = CollapsedPointSource('cps-%d-%d' % (task_no, i), ps[idxs])
            cps.id = ps[0].id
            cps.grp_id = ps[0].grp_id
            cps.trt_smr = ps[0].trt_smr
            out.append(cps)
        else:  # there is a single source
            out.append(ps[idxs[0]])
    return {grp_id: out}
Ejemplo n.º 4
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    if getattr(group, 'src_interdep', None) == 'mutex':
        mutex_weight = {
            src.source_id: weight
            for src, weight in zip(group.sources, group.srcs_weights)
        }
    else:
        mutex_weight = None
    grp_ids = set()
    for src in group:
        grp_ids.update(src.src_group_ids)
    maxdist = src_filter.integration_distance
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = param['imtls']
        trunclevel = param.get('truncation_level')
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('make_contexts', measuremem=False)
        poe_mon = monitor('get_poes', measuremem=False)
        pmap = AccumDict({
            grp_id: ProbabilityMap(len(imtls.array), len(gsims))
            for grp_id in grp_ids
        })
        # AccumDict of arrays with 4 elements weight, nsites, calc_time, split
        pmap.calc_times = AccumDict(accum=numpy.zeros(4))
        pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
        for src, s_sites in src_filter(group):  # filter now
            t0 = time.time()
            indep = group.rup_interdep == 'indep' if mutex_weight else True
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon,
                                    poe_mon, indep)
            if mutex_weight:  # mutex sources
                weight = mutex_weight[src.source_id]
                for sid in poemap:
                    pcurve = pmap[group.id].setdefault(sid, 0)
                    pcurve += poemap[sid] * weight
            elif poemap:
                for grp_id in src.src_group_ids:
                    pmap[grp_id] |= poemap
            src_id = src.source_id.split(':', 1)[0]
            pmap.calc_times[src_id] += numpy.array(
                [src.weight, len(s_sites),
                 time.time() - t0, 1])
            # storing the number of contributing ruptures too
            pmap.eff_ruptures += {
                grp_id: getattr(poemap, 'eff_ruptures', 0)
                for grp_id in src.src_group_ids
            }
        if mutex_weight and group.grp_probability is not None:
            pmap[group.id] *= group.grp_probability
        return pmap
Ejemplo n.º 5
0
 def __init__(self, oqparam, calc_id=None):
     self.datastore = datastore.DataStore(calc_id)
     self._monitor = Monitor(
         '%s.run' % self.__class__.__name__, measuremem=True)
     self.oqparam = oqparam
     if 'performance_data' not in self.datastore:
         self.datastore.create_dset('performance_data', perf_dt)
Ejemplo n.º 6
0
def safely_call(func, args, pickle=False):
    """
    Call the given function with the given arguments safely, i.e.
    by trapping the exceptions. Return a pair (result, exc_type)
    where exc_type is None if no exceptions occur, otherwise it
    is the exception class and the result is a string containing
    error message and traceback.

    :param func: the function to call
    :param args: the arguments
    :param pickle:
        if set, the input arguments are unpickled and the return value
        is pickled; otherwise they are left unchanged
    """
    if pickle:
        args = [a.unpickle() for a in args]
    ismon = args and isinstance(args[-1], Monitor)
    mon = args[-1] if ismon else Monitor()
    try:
        got = func(*args)
        if inspect.isgenerator(got):
            got = list(got)
        res = got, None, mon
    except:
        etype, exc, tb = sys.exc_info()
        tb_str = ''.join(traceback.format_tb(tb))
        res = ('\n%s%s: %s' % (tb_str, etype.__name__, exc), etype, mon)
    if pickle:
        return Pickled(res)
    return res
Ejemplo n.º 7
0
def make_gmfs(eb_ruptures, sitecol, imts, gsims,
              trunc_level, correl_model, monitor=Monitor()):
    """
    :param eb_ruptures: a list of EBRuptures
    :param sitecol: a SiteCollection instance
    :param imts: an ordered list of intensity measure type strings
    :param gsims: an order list of GSIM instance
    :param trunc_level: truncation level
    :param correl_model: correlation model instance
    :param monitor: a monitor instance
    :returns: a dictionary serial -> GmfaSidsEtags
    """
    dic = {}  # serial -> GmfaSidsEtags
    ctx_mon = monitor('make contexts')
    gmf_mon = monitor('compute poes')
    sites = sitecol.complete
    for ebr in eb_ruptures:
        with ctx_mon:
            r_sites = site.FilteredSiteCollection(ebr.indices, sites)
            computer = calc.gmf.GmfComputer(
                ebr.rupture, r_sites, imts, gsims, trunc_level, correl_model)
        with gmf_mon:
            gmfa = computer.calcgmfs(ebr.multiplicity, ebr.rupture.seed)
            dic[ebr.serial] = GmfaSidsEtags(gmfa, r_sites.indices, ebr.etags)
    return dic
Ejemplo n.º 8
0
 def __init__(self, task_func, task_args=(), monitor=None, distribute=None,
              progress=logging.info):
     self.__class__.init(distribute=distribute or OQ_DISTRIBUTE)
     self.task_func = task_func
     self.monitor = monitor or Monitor(task_func.__name__)
     self.calc_id = getattr(self.monitor, 'calc_id', None)
     self.name = self.monitor.operation or task_func.__name__
     self.task_args = task_args
     self.distribute = distribute or oq_distribute(task_func)
     self.progress = progress
     try:
         self.num_tasks = len(self.task_args)
     except TypeError:  # generators have no len
         self.num_tasks = None
     # a task can be a function, a class or an instance with a __call__
     if inspect.isfunction(task_func):
         self.argnames = inspect.getfullargspec(task_func).args
     elif inspect.isclass(task_func):
         self.argnames = inspect.getfullargspec(task_func.__init__).args[1:]
     else:  # instance with a __call__ method
         self.argnames = inspect.getfullargspec(task_func.__call__).args[1:]
     self.receiver = 'tcp://%s:%s' % (
         config.dbserver.listen, config.dbserver.receiver_ports)
     self.sent = numpy.zeros(len(self.argnames) - 1)
     self.monitor.backurl = None  # overridden later
     self.tasks = []  # populated by .submit
     h5 = self.monitor.hdf5
     task_info = 'task_info/' + self.name
     if h5 and task_info not in h5:  # first time
         # task_info and performance_data should be generated in advance
         hdf5.create(h5, task_info, task_info_dt)
     if h5 and 'performance_data' not in h5:
         hdf5.create(h5, 'performance_data', perf_dt)
Ejemplo n.º 9
0
def collect_bins_data(trt_num,
                      sources,
                      site,
                      cmaker,
                      imldict,
                      truncation_level,
                      n_epsilons,
                      mon=Monitor()):
    sitecol = SiteCollection([site])
    # NB: instantiating truncnorm is slow and calls the infamous "doccer"
    truncnorm = scipy.stats.truncnorm(-truncation_level, truncation_level)
    acc = AccumDict(accum=[])
    for source in sources:
        try:
            trti = trt_num[source.tectonic_region_type]
            rupdict = cmaker.disaggregate(sitecol, source.iter_ruptures(),
                                          imldict, truncnorm, n_epsilons, mon)
            acc['trti'].extend([trti] * len(rupdict['mags']))
            acc += rupdict
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, err)
            raise_(etype, msg, tb)
    return acc
Ejemplo n.º 10
0
 def init(cls, poolsize=None, distribute=OQ_DISTRIBUTE):
     if distribute == 'processpool' and not hasattr(cls, 'pool'):
         cls.pool = multiprocessing.Pool(poolsize, init_workers)
         m = Monitor('wakeup')
         cls(_wakeup, [(.2, m) for _ in range(cls.pool._processes)])
     elif distribute == 'threadpool' and not hasattr(cls, 'pool'):
         cls.pool = multiprocessing.dummy.Pool(poolsize)
Ejemplo n.º 11
0
def info(calculators, gsims, views, exports, report, input_file=''):
    """
    Give information. You can pass the name of an available calculator,
    a job.ini file, or a zip archive with the input files.
    """
    logging.basicConfig(level=logging.INFO)
    if calculators:
        for calc in sorted(base.calculators):
            print(calc)
    if gsims:
        for gs in gsim.get_available_gsims():
            print(gs)
    if views:
        for name in sorted(datastore.view):
            print(name)
    if exports:
        dic = groupby(export, operator.itemgetter(0),
                      lambda group: [r[1] for r in group])
        n = 0
        for exporter, formats in dic.items():
            print(exporter, formats)
            n += len(formats)
        print('There are %d exporters defined.' % n)
    if input_file.endswith('.xml'):
        print(nrml.read(input_file).to_str())
    elif input_file.endswith(('.ini', '.zip')):
        with Monitor('info', measuremem=True) as mon:
            if report:
                print('Generated', reportwriter.build_report(input_file))
            else:
                print_csm_info(input_file)
        if mon.duration > 1:
            print(mon)
    elif input_file:
        print("No info for '%s'" % input_file)
Ejemplo n.º 12
0
def collect_bin_data(sources, sitecol, cmaker, iml4,
                     truncation_level, n_epsilons, monitor=Monitor()):
    """
    :param sources: a list of sources
    :param sitecol: a SiteCollection instance
    :param cmaker: a ContextMaker instance
    :param iml4: an ArrayWrapper of intensities of shape (N, R, M, P)
    :param truncation_level: the truncation level
    :param n_epsilons: the number of epsilons
    :param monitor: a Monitor instance
    :returns: a dictionary (poe, imt, rlzi) -> probabilities of shape (N, E)
    """
    # NB: instantiating truncnorm is slow and calls the infamous "doccer"
    truncnorm = scipy.stats.truncnorm(-truncation_level, truncation_level)
    epsilons = numpy.linspace(truncnorm.a, truncnorm.b, n_epsilons + 1)
    acc = AccumDict(accum=[])
    for source in sources:
        ruptures = source.iter_ruptures()
        try:
            acc += cmaker.disaggregate(
                sitecol, ruptures, iml4, truncnorm, epsilons, monitor)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = 'An error occurred with source id=%s. Error: %s'
            msg %= (source.source_id, err)
            raise_(etype, msg, tb)
    return pack(acc, 'mags dists lons lats'.split())
Ejemplo n.º 13
0
def sample_ruptures(sources, srcfilter, param, monitor=Monitor()):
    """
    :param sources:
        a sequence of sources of the same group
    :param srcfilter:
        SourceFilter instance used also for bounding box post filtering
    :param param:
        a dictionary of additional parameters including
        ses_per_logic_tree_path
    :param monitor:
        monitor instance
    :yields:
        dictionaries with keys rup_array, calc_times
    """
    # AccumDict of arrays with 3 elements num_ruptures, num_sites, calc_time
    calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
    # Compute and save stochastic event sets
    num_ses = param['ses_per_logic_tree_path']
    [grp_id] = set(src.src_group_id for src in sources)
    # Compute the number of occurrences of the source group. This is used
    # for cluster groups or groups with mutually exclusive sources.
    if (getattr(sources, 'atomic', False) and
            getattr(sources, 'cluster', False)):
        eb_ruptures, calc_times = sample_cluster(
            sources, srcfilter, num_ses, param)

        # Yield ruptures
        yield AccumDict(rup_array=get_rup_array(eb_ruptures, srcfilter),
                        calc_times=calc_times,
                        eff_ruptures={grp_id: len(eb_ruptures)})
    else:
        eb_ruptures = []
        eff_ruptures = 0
        # AccumDict of arrays with 2 elements weight, calc_time
        calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
        for src, _sites in srcfilter(sources):
            eff_ruptures += 1
            t0 = time.time()
            if len(eb_ruptures) > MAX_RUPTURES:
                # yield partial result to avoid running out of memory
                yield AccumDict(rup_array=get_rup_array(eb_ruptures,
                                                        srcfilter),
                                calc_times={}, eff_ruptures={})
                eb_ruptures.clear()
            samples = getattr(src, 'samples', 1)
            n_occ = 0
            for rup, n_occ in src.sample_ruptures(samples * num_ses):
                ebr = EBRupture(rup, src.id, grp_id, n_occ, samples)
                eb_ruptures.append(ebr)
                n_occ += ebr.n_occ
            dt = time.time() - t0
            try:
                n_sites = len(_sites)
            except (TypeError, ValueError):  # for None or a closed dataset
                n_sites = 0
            calc_times[src.id] += numpy.array([n_occ, n_sites, dt])
        rup_array = get_rup_array(eb_ruptures, srcfilter)
        yield AccumDict(rup_array=rup_array, calc_times=calc_times,
                        eff_ruptures={grp_id: eff_ruptures})
Ejemplo n.º 14
0
def sample_ruptures(sources, srcfilter, param, monitor=Monitor()):
    """
    :param sources:
        a sequence of sources of the same group
    :param srcfilter:
        SourceFilter instance used also for bounding box post filtering
    :param param:
        a dictionary of additional parameters including
        ses_per_logic_tree_path
    :param monitor:
        monitor instance
    :yields:
        dictionaries with keys rup_array, calc_times
    """
    # AccumDict of arrays with 3 elements num_ruptures, num_sites, calc_time
    calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
    # Compute and save stochastic event sets
    num_ses = param['ses_per_logic_tree_path']
    trt = sources[0].tectonic_region_type
    # Compute the number of occurrences of the source group. This is used
    # for cluster groups or groups with mutually exclusive sources.
    if (getattr(sources, 'atomic', False)
            and getattr(sources, 'cluster', False)):
        eb_ruptures, calc_times = sample_cluster(sources, srcfilter, num_ses,
                                                 param)

        # Yield ruptures
        yield AccumDict(
            dict(rup_array=get_rup_array(eb_ruptures, srcfilter),
                 calc_times=calc_times,
                 eff_ruptures={trt: len(eb_ruptures)}))
    else:
        eb_ruptures = []
        eff_ruptures = 0
        # AccumDict of arrays with 2 elements weight, calc_time
        calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
        for src, _ in srcfilter.filter(sources):
            nr = src.num_ruptures
            eff_ruptures += nr
            t0 = time.time()
            if len(eb_ruptures) > MAX_RUPTURES:
                # yield partial result to avoid running out of memory
                yield AccumDict(
                    dict(rup_array=get_rup_array(eb_ruptures, srcfilter),
                         calc_times={},
                         eff_ruptures={}))
                eb_ruptures.clear()
            samples = getattr(src, 'samples', 1)
            for rup, trt_smrlz, n_occ in src.sample_ruptures(
                    samples * num_ses, param['ses_seed']):
                ebr = EBRupture(rup, src.source_id, trt_smrlz, n_occ)
                eb_ruptures.append(ebr)
            dt = time.time() - t0
            calc_times[src.id] += numpy.array([nr, src.nsites, dt])
        rup_array = get_rup_array(eb_ruptures, srcfilter)
        yield AccumDict(
            dict(rup_array=rup_array,
                 calc_times=calc_times,
                 eff_ruptures={trt: eff_ruptures}))
Ejemplo n.º 15
0
 def test(self):
     mon = Monitor()
     iterargs = ((i, mon) for i in range(10))
     res = _starmap(double, iterargs, self.host, self.task_in_port,
                    self.receiver_ports)
     num_tasks = next(res)
     self.assertEqual(num_tasks, 10)
     self.assertEqual(sum(r[0] for r in res), 90)
Ejemplo n.º 16
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times,
        .eff_ruptures
    """
    grp_ids = set()
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures is set when parsing the XML, but not when
            # the source is instantiated manually, so it is set here
            src.num_ruptures = src.count_ruptures()
        grp_ids.update(src.src_group_ids)
    maxdist = src_filter.integration_distance
    imtls = param['imtls']
    trunclevel = param.get('truncation_level')
    cmaker = ContextMaker(gsims, maxdist, param, monitor)
    pmap = AccumDict({
        grp_id: ProbabilityMap(len(imtls.array), len(gsims))
        for grp_id in grp_ids
    })
    # AccumDict of arrays with 3 elements weight, nsites, calc_time
    pmap.calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
    pmap.eff_ruptures = AccumDict()  # grp_id -> num_ruptures
    src_mutex = param.get('src_interdep') == 'mutex'
    rup_mutex = param.get('rup_interdep') == 'mutex'
    for src, s_sites in src_filter(group):  # filter now
        t0 = time.time()
        try:
            poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel,
                                    not rup_mutex)
        except Exception as err:
            etype, err, tb = sys.exc_info()
            msg = '%s (source id=%s)' % (str(err), src.source_id)
            raise etype(msg).with_traceback(tb)
        if src_mutex:  # mutex sources, there is a single group
            for sid in poemap:
                pcurve = pmap[src.src_group_id].setdefault(sid, 0)
                pcurve += poemap[sid] * src.mutex_weight
        elif poemap:
            for gid in src.src_group_ids:
                pmap[gid] |= poemap
        pmap.calc_times[src.id] += numpy.array(
            [src.weight, len(s_sites),
             time.time() - t0])
        # storing the number of contributing ruptures too
        pmap.eff_ruptures += {
            gid: getattr(poemap, 'eff_ruptures', 0)
            for gid in src.src_group_ids
        }
    if src_mutex and param.get('grp_probability'):
        pmap[src.src_group_id] *= param['grp_probability']
    return pmap
Ejemplo n.º 17
0
def sample_ruptures(group,
                    src_filter=filters.source_site_noop_filter,
                    gsims=(),
                    param=(),
                    monitor=Monitor()):
    """
    :param group:
        a SourceGroup or a sequence of sources of the same group
    :param src_filter:
        a source site filter (default noop filter)
    :param gsims:
        a list of GSIMs for the current tectonic region model
    :param param:
        a dictionary of additional parameters (by default
        ses_per_logic_tree_path=1,  samples=1, seed=42, filter_distance=1000)
    :param monitor:
        monitor instance
    :returns:
        a dictionary with eb_ruptures, num_events, num_ruptures, calc_times
    """
    if not param:
        param = dict(ses_per_logic_tree_path=1,
                     samples=1,
                     seed=42,
                     filter_distance=1000)
    if getattr(group, 'src_interdep', None) == 'mutex':
        prob = {src: sw for src, sw in zip(group, group.srcs_weights)}
    else:
        prob = {src: 1 for src in group}
    eb_ruptures = []
    calc_times = []
    rup_mon = monitor('making contexts', measuremem=False)
    # Compute and save stochastic event sets
    num_ruptures = 0
    eids = numpy.zeros(0)
    cmaker = ContextMaker(gsims, src_filter.integration_distance,
                          param['filter_distance'], monitor)
    for src, s_sites in src_filter(group):
        t0 = time.time()
        num_ruptures += src.num_ruptures
        num_occ_by_rup = _sample_ruptures(src, prob[src],
                                          param['ses_per_logic_tree_path'],
                                          param['samples'], param['seed'])
        # NB: the number of occurrences is very low, << 1, so it is
        # more efficient to filter only the ruptures that occur, i.e.
        # to call sample_ruptures *before* the filtering
        for ebr in _build_eb_ruptures(src, num_occ_by_rup, cmaker, s_sites,
                                      param['seed'], rup_mon):
            eb_ruptures.append(ebr)
        eids = set_eids(eb_ruptures)
        src_id = src.source_id.split(':', 1)[0]
        dt = time.time() - t0
        calc_times.append((src_id, src.nsites, eids, dt))
    dic = dict(eb_ruptures=eb_ruptures,
               num_events=len(eids),
               calc_times=calc_times,
               num_ruptures=num_ruptures)
    return dic
Ejemplo n.º 18
0
def info(calculators, gsims, views, exports, report, input_file=''):
    """
    Give information. You can pass the name of an available calculator,
    a job.ini file, or a zip archive with the input files.
    """
    logging.basicConfig(level=logging.INFO)
    if calculators:
        for calc in sorted(base.calculators):
            print(calc)
    if gsims:
        for gs in gsim.get_available_gsims():
            print(gs)
    if views:
        for name in sorted(view):
            print(name)
    if exports:
        dic = groupby(export, operator.itemgetter(0),
                      lambda group: [r[1] for r in group])
        n = 0
        for exporter, formats in dic.items():
            print(exporter, formats)
            n += len(formats)
        print('There are %d exporters defined.' % n)
    if os.path.isdir(input_file) and report:
        with Monitor('info', measuremem=True) as mon:
            with mock.patch.object(logging.root, 'info'):  # reduce logging
                do_build_reports(input_file)
        print(mon)
    elif input_file.endswith('.xml'):
        node = nrml.read(input_file)
        if node[0].tag.endswith('sourceModel'):
            assert node['xmlns'].endswith('nrml/0.5'), node['xmlns']
            print(source_model_info(node[0]))
        else:
            print(node.to_str())
    elif input_file.endswith(('.ini', '.zip')):
        with Monitor('info', measuremem=True) as mon:
            if report:
                print('Generated', reportwriter.build_report(input_file))
            else:
                print_csm_info(input_file)
        if mon.duration > 1:
            print(mon)
    elif input_file:
        print("No info for '%s'" % input_file)
Ejemplo n.º 19
0
def classical(group, src_filter, gsims, param, monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns:
        a dictionary with keys pmap, calc_times, rup_data, extra
    """
    if not hasattr(src_filter, 'sitecol'):  # do not filter
        src_filter = SourceFilter(src_filter, {})

    # Get the parameters assigned to the group
    src_mutex = getattr(group, 'src_interdep', None) == 'mutex'
    cluster = getattr(group, 'cluster', None)
    trts = set()
    maxradius = 0
    for src in group:
        if not src.num_ruptures:
            # src.num_ruptures may not be set, so it is set here
            src.num_ruptures = src.count_ruptures()
        # set the proper TOM in case of a cluster
        if cluster:
            src.temporal_occurrence_model = FatedTOM(time_span=1)
        trts.add(src.tectonic_region_type)
        if hasattr(src, 'radius'):  # for prefiltered point sources
            maxradius = max(maxradius, src.radius)

    param['maximum_distance'] = src_filter.integration_distance
    [trt] = trts  # there must be a single tectonic region type
    cmaker = ContextMaker(trt, gsims, param, monitor)
    try:
        cmaker.tom = group.temporal_occurrence_model
    except AttributeError:  # got a list of sources, not a group
        time_span = param.get('investigation_time')  # None for nonparametric
        cmaker.tom = PoissonTOM(time_span) if time_span else None
    if cluster:
        cmaker.tom = FatedTOM(time_span=1)
    pmap, rup_data, calc_times = PmapMaker(cmaker, src_filter, group).make()
    extra = {}
    extra['task_no'] = getattr(monitor, 'task_no', 0)
    extra['trt'] = trt
    extra['source_id'] = src.source_id
    extra['grp_id'] = src.grp_id
    extra['maxradius'] = maxradius
    group_probability = getattr(group, 'grp_probability', None)
    if src_mutex and group_probability:
        pmap *= group_probability

    if cluster:
        tom = getattr(group, 'temporal_occurrence_model')
        pmap = _cluster(param['imtls'], tom, gsims, pmap)
    return dict(pmap=pmap,
                calc_times=calc_times,
                rup_data=rup_data,
                extra=extra)
Ejemplo n.º 20
0
 def __init__(self, oqparam, calc_id):
     self.datastore = datastore.DataStore(calc_id)
     init_performance(self.datastore.hdf5)
     self._monitor = Monitor('%s.run' % self.__class__.__name__,
                             measuremem=True,
                             h5=self.datastore)
     # NB: using h5=self.datastore.hdf5 would mean losing the performance
     # info about Calculator.run since the file will be closed later on
     self.oqparam = oqparam
Ejemplo n.º 21
0
def run_calc(job_id,
             oqparam,
             log_level,
             log_file,
             exports,
             hazard_calculation_id=None):
    """
    Run a calculation.

    :param job_id:
        ID of the current job
    :param oqparam:
        :class:`openquake.commonlib.oqvalidation.OqParam` instance
    :param str log_level:
        The desired logging level. Valid choices are 'debug', 'info',
        'progress', 'warn', 'error', and 'critical'.
    :param str log_file:
        Complete path (including file name) to file where logs will be written.
        If `None`, logging will just be printed to standard output.
    :param exports:
        A comma-separated string of export types.
    """
    monitor = Monitor('total runtime', measuremem=True)
    with logs.handle(job_id, log_level, log_file):  # run the job
        if USE_CELERY and os.environ.get('OQ_DISTRIBUTE') == 'celery':
            set_concurrent_tasks_default()
        calc = base.calculators(oqparam, monitor, calc_id=job_id)
        tb = 'None\n'
        try:
            _do_run_calc(calc, exports, hazard_calculation_id)
            logs.dbcmd('finish', job_id, 'complete')
            expose_outputs(calc.datastore)
            records = views.performance_view(calc.datastore)
            logs.dbcmd('save_performance', job_id, records)
            calc.datastore.close()
            logs.LOG.info('Calculation %d finished correctly in %d seconds',
                          job_id, calc.monitor.duration)
        except:
            tb = traceback.format_exc()
            try:
                logs.LOG.critical(tb)
                logs.dbcmd('finish', job_id, 'failed')
            except:  # an OperationalError may always happen
                sys.stderr.write(tb)
            raise
        finally:
            # if there was an error in the calculation, this part may fail;
            # in such a situation, we simply log the cleanup error without
            # taking further action, so that the real error can propagate
            try:
                if USE_CELERY:
                    celery_cleanup(TERMINATE, parallel.TaskManager.task_ids)
            except:
                # log the finalization error only if there is no real error
                if tb == 'None\n':
                    logs.LOG.error('finalizing', exc_info=True)
    return calc
Ejemplo n.º 22
0
def sample_ruptures(sources, cmaker, sitecol=None, monitor=Monitor()):
    """
    :param sources:
        a sequence of sources of the same group
    :param cmaker:
        a ContextMaker instance with ses_per_logic_tree_path, ses_seed
    :param sitecol:
        SiteCollection instance used for filtering (None for no filtering)
    :param monitor:
        monitor instance
    :yields:
        dictionaries with keys rup_array, calc_times
    """
    srcfilter = SourceFilter(sitecol, cmaker.maximum_distance)
    # AccumDict of arrays with 3 elements num_ruptures, num_sites, calc_time
    calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
    # Compute and save stochastic event sets
    num_ses = cmaker.ses_per_logic_tree_path
    grp_id = sources[0].grp_id
    # Compute the number of occurrences of the source group. This is used
    # for cluster groups or groups with mutually exclusive sources.
    if (getattr(sources, 'atomic', False) and
            getattr(sources, 'cluster', False)):
        eb_ruptures, calc_times = sample_cluster(
            sources, srcfilter, num_ses, vars(cmaker))

        # Yield ruptures
        er = sum(src.num_ruptures for src, _ in srcfilter.filter(sources))
        yield AccumDict(dict(rup_array=get_rup_array(eb_ruptures, srcfilter),
                             calc_times=calc_times, eff_ruptures={grp_id: er}))
    else:
        eb_ruptures = []
        eff_ruptures = 0
        # AccumDict of arrays with 2 elements weight, calc_time
        calc_times = AccumDict(accum=numpy.zeros(3, numpy.float32))
        for src, _ in srcfilter.filter(sources):
            nr = src.num_ruptures
            eff_ruptures += nr
            t0 = time.time()
            if len(eb_ruptures) > MAX_RUPTURES:
                # yield partial result to avoid running out of memory
                yield AccumDict(dict(rup_array=get_rup_array(eb_ruptures,
                                                             srcfilter),
                                     calc_times={}, eff_ruptures={}))
                eb_ruptures.clear()
            samples = getattr(src, 'samples', 1)
            for rup, trt_smr, n_occ in src.sample_ruptures(
                    samples * num_ses, cmaker.ses_seed):
                ebr = EBRupture(rup, src.source_id, trt_smr, n_occ)
                eb_ruptures.append(ebr)
            dt = time.time() - t0
            calc_times[src.id] += numpy.array([nr, src.nsites, dt])
        rup_array = get_rup_array(eb_ruptures, srcfilter)
        yield AccumDict(dict(rup_array=rup_array, calc_times=calc_times,
                             eff_ruptures={grp_id: eff_ruptures}))
Ejemplo n.º 23
0
def pmap_from_grp(sources,
                  source_site_filter,
                  imtls,
                  gsims,
                  truncation_level=None,
                  bbs=(),
                  monitor=Monitor()):
    """
    Compute the hazard curves for a set of sources belonging to the same
    tectonic region type for all the GSIMs associated to that TRT.
    The arguments are the same as in :func:`calc_hazard_curves`, except
    for ``gsims``, which is a list of GSIM instances.

    :returns: a ProbabilityMap instance
    """
    if isinstance(sources, SourceGroup):
        group = sources
        sources = group.sources
        trt = sources[0].tectonic_region_type
    else:  # list of sources
        trt = sources[0].tectonic_region_type
        group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep')
    try:
        maxdist = source_site_filter.integration_distance[trt]
    except:
        maxdist = source_site_filter.integration_distance
    if hasattr(gsims, 'keys'):  # dictionary trt -> gsim
        gsims = [gsims[trt]]
    with GroundShakingIntensityModel.forbid_instantiation():
        imtls = DictArray(imtls)
        cmaker = ContextMaker(gsims, maxdist)
        ctx_mon = monitor('making contexts', measuremem=False)
        pne_mon = monitor('computing poes', measuremem=False)
        disagg_mon = monitor('get closest points', measuremem=False)
        src_indep = group.src_interdep == 'indep'
        pmap = ProbabilityMap(len(imtls.array), len(gsims))
        pmap.calc_times = []  # pairs (src_id, delta_t)
        pmap.grp_id = sources[0].src_group_id
        for src, s_sites in source_site_filter(sources):
            t0 = time.time()
            poemap = poe_map(src, s_sites, imtls, cmaker, truncation_level,
                             bbs, group.rup_interdep == 'indep', ctx_mon,
                             pne_mon, disagg_mon)
            if src_indep:  # usual composition of probabilities
                pmap |= poemap
            else:  # mutually exclusive probabilities
                weight = float(group.srcs_weights[src.source_id])
                for sid in poemap:
                    pmap[sid] += poemap[sid] * weight
            pmap.calc_times.append(
                (src.source_id, len(s_sites), time.time() - t0))
        # storing the number of contributing ruptures too
        pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts}
        return pmap
Ejemplo n.º 24
0
 def get_hazard(self, rlzs_assoc, monitor=Monitor()):
     """
     :param rlzs_assoc:
         :class:`openquake.commonlib.source.RlzsAssoc` instance
     :param monitor:
         a :class:`openquake.baselib.performance.Monitor` instance
     :returns:
         list of hazard dictionaries imt -> rlz -> haz per each site
     """
     return [{
         self.imt: rlzs_assoc.combine(hazard)
     } for hazard in self.hazard_by_site]
Ejemplo n.º 25
0
    def disaggregate(self,
                     sitecol,
                     ruptures,
                     iml4,
                     truncnorm,
                     epsilons,
                     monitor=Monitor()):
        """
        Disaggregate (separate) PoE of `imldict` in different contributions
        each coming from `n_epsilons` distribution bins.

        :param sitecol: a SiteCollection
        :param ruptures: an iterator over ruptures with the same TRT
        :param iml4: a 4d array of IMLs of shape (N, R, M, P)
        :param truncnorm: an instance of scipy.stats.truncnorm
        :param epsilons: the epsilon bins
        :param monitor: a Monitor instance
        :returns: an AccumDict
        """
        sitemesh = sitecol.mesh
        acc = AccumDict(accum=[])
        ctx_mon = monitor('make_contexts', measuremem=False)
        pne_mon = monitor('disaggregate_pne', measuremem=False)
        for rupture in ruptures:
            with ctx_mon:
                sctx, rctx, orig_dctx = self.make_contexts(sitecol,
                                                           rupture,
                                                           filter=False)
                if (self.maximum_distance
                        and orig_dctx.rjb.min() > self.maximum_distance(
                            rupture.tectonic_region_type, rupture.mag)):
                    continue  # rupture away from all sites
            cache = {}
            for r, gsim in self.gsim_by_rlzi.items():
                dctx = orig_dctx.roundup(gsim.minimum_distance)
                for m, imt in enumerate(iml4.imts):
                    for p, poe in enumerate(iml4.poes_disagg):
                        iml = tuple(iml4.array[:, r, m, p])
                        try:
                            pne = cache[gsim, imt, iml]
                        except KeyError:
                            with pne_mon:
                                pne = gsim.disaggregate_pne(
                                    rupture, sctx, rctx, dctx, imt, iml,
                                    truncnorm, epsilons)
                                cache[gsim, imt, iml] = pne
                        acc[poe, str(imt), r].append(pne)
            closest_points = rupture.surface.get_closest_points(sitemesh)
            acc['mags'].append(rupture.mag)
            acc['dists'].append(dctx.rjb)
            acc['lons'].append(closest_points.lons)
            acc['lats'].append(closest_points.lats)
        return acc
Ejemplo n.º 26
0
def main(dirname):
    dname = pathlib.Path(dirname)
    with hdf5new() as hdf5:  # create a new datastore
        monitor = Monitor('count', hdf5)  # create a new monitor
        iterargs = ((open(dname/fname, encoding='utf-8').read(),)
                    for fname in os.listdir(dname)
                    if fname.endswith('.rst'))  # read the docs
        c = collections.Counter()  # intially empty counter
        for counter in Starmap(count, iterargs, monitor):
            c += counter
        print(c)  # total counts
        print('Performance info stored in', hdf5)
Ejemplo n.º 27
0
    def disaggregate(self,
                     sitecol,
                     ruptures,
                     iml4,
                     truncnorm,
                     epsilons,
                     monitor=Monitor()):
        """
        Disaggregate (separate) PoE of `imldict` in different contributions
        each coming from `n_epsilons` distribution bins.

        :param sitecol: a SiteCollection
        :param ruptures: an iterator over ruptures with the same TRT
        :param iml4: a 4d array of IMLs of shape (N, R, M, P)
        :param truncnorm: an instance of scipy.stats.truncnorm
        :param epsilons: the epsilon bins
        :param monitor: a Monitor instance
        :returns:
            an AccumDict with keys (poe, imt, rlzi) and mags, dists, lons, lats
        """
        acc = AccumDict(accum=[])
        ctx_mon = monitor('disagg_contexts', measuremem=False)
        pne_mon = monitor('disaggregate_pne', measuremem=False)
        clo_mon = monitor('get_closest', measuremem=False)
        for rupture in ruptures:
            with ctx_mon:
                orig_dctx = DistancesContext(
                    (param, get_distances(rupture, sitecol, param))
                    for param in self.REQUIRES_DISTANCES)
                self.add_rup_params(rupture)
            with clo_mon:  # this is faster than computing orig_dctx
                closest_points = rupture.surface.get_closest_points(sitecol)
            cache = {}
            for r, gsim in self.gsim_by_rlzi.items():
                dctx = orig_dctx.roundup(gsim.minimum_distance)
                for m, imt in enumerate(iml4.imts):
                    for p, poe in enumerate(iml4.poes_disagg):
                        iml = tuple(iml4.array[:, r, m, p])
                        try:
                            pne = cache[gsim, imt, iml]
                        except KeyError:
                            with pne_mon:
                                pne = gsim.disaggregate_pne(
                                    rupture, sitecol, dctx, imt, iml,
                                    truncnorm, epsilons)
                                cache[gsim, imt, iml] = pne
                        acc[poe, str(imt), r].append(pne)
            acc['mags'].append(rupture.mag)
            acc['dists'].append(getattr(dctx, self.filter_distance))
            acc['lons'].append(closest_points.lons)
            acc['lats'].append(closest_points.lats)
        return acc
Ejemplo n.º 28
0
    def __init__(self, trt, gsims, param=None, monitor=Monitor()):
        param = param or {}  # empty in the gmpe-smtk
        self.af = param.get('af', None)
        self.max_sites_disagg = param.get('max_sites_disagg', 10)
        self.collapse_level = param.get('collapse_level', False)
        self.trt = trt
        self.gsims = gsims
        self.single_site_opt = numpy.array(
            [hasattr(gsim, 'get_mean_std1') for gsim in gsims])
        self.maximum_distance = (param.get('maximum_distance')
                                 or MagDepDistance({}))
        self.investigation_time = param.get('investigation_time')
        self.trunclevel = param.get('truncation_level')
        self.num_epsilon_bins = param.get('num_epsilon_bins', 1)
        self.grp_id = param.get('grp_id', 0)
        self.effect = param.get('effect')
        self.task_no = getattr(monitor, 'task_no', 0)
        for req in self.REQUIRES:
            reqset = set()
            for gsim in gsims:
                reqset.update(getattr(gsim, 'REQUIRES_' + req))
            setattr(self, 'REQUIRES_' + req, reqset)
        # self.pointsource_distance is a dict mag -> dist, possibly empty
        psd = param.get('pointsource_distance')
        if hasattr(psd, 'ddic'):
            self.pointsource_distance = psd.ddic.get(trt, {})
        else:
            self.pointsource_distance = {}
        if 'imtls' in param:
            self.imtls = param['imtls']
        elif 'hazard_imtls' in param:
            self.imtls = DictArray(param['hazard_imtls'])
        else:
            self.imtls = {}
        self.imts = [imt_module.from_string(imt) for imt in self.imtls]
        self.reqv = param.get('reqv')
        if self.reqv is not None:
            self.REQUIRES_DISTANCES.add('repi')
        self.mon = monitor
        self.ctx_mon = monitor('make_contexts', measuremem=False)
        self.loglevels = DictArray(self.imtls) if self.imtls else {}
        self.shift_hypo = param.get('shift_hypo')
        with warnings.catch_warnings():
            # avoid RuntimeWarning: divide by zero encountered in log
            warnings.simplefilter("ignore")
            for imt, imls in self.imtls.items():
                if imt != 'MMI':
                    self.loglevels[imt] = numpy.log(imls)

        # instantiate monitors
        self.gmf_mon = monitor('computing mean_std', measuremem=False)
        self.poe_mon = monitor('get_poes', measuremem=False)
Ejemplo n.º 29
0
 def get_hazard(self, rlzs_assoc, monitor=Monitor()):
     """
     :param rlzs_assoc:
         :class:`openquake.commonlib.source.RlzsAssoc` instance
     :param monitor:
         a :class:`openquake.baselib.performance.Monitor` instance
     :returns:
         lists of N hazard dictionaries imt -> rlz -> Gmvs
     """
     gmfcoll = create(GmfCollector, self.ses_ruptures, self.sitecol,
                      self.imts, rlzs_assoc, self.trunc_level,
                      self.correl_model, self.min_iml, monitor)
     return gmfcoll
Ejemplo n.º 30
0
def disaggregate(cmaker,
                 sitecol,
                 rupdata,
                 iml2,
                 truncnorm,
                 epsilons,
                 monitor=Monitor()):
    """
    Disaggregate (separate) PoE in different contributions.

    :param cmaker: a ContextMaker instance
    :param sitecol: a SiteCollection with N=1 site
    :param ruptures: an iterator over ruptures with the same TRT
    :param iml2: a 2D array of IMLs of shape (M, P)
    :param truncnorm: an instance of scipy.stats.truncnorm
    :param epsilons: the epsilon bins
    :param monitor: a Monitor instance
    :returns:
        an AccumDict with keys (poe, imt, rlzi) and mags, dists, lons, lats
    """
    assert len(sitecol) == 1, sitecol
    acc = AccumDict(accum=[], mags=[], dists=[], lons=[], lats=[])
    try:
        gsim = cmaker.gsim_by_rlzi[iml2.rlzi]
    except KeyError:
        return acc
    pne_mon = monitor('disaggregate_pne', measuremem=False)
    [sid] = sitecol.sids
    acc['mags'] = rupdata['mag']
    acc['lons'] = rupdata['lon'][:, sid]
    acc['lats'] = rupdata['lat'][:, sid]
    acc['dists'] = dists = rupdata[cmaker.filter_distance][:, sid]
    if gsim.minimum_distance:
        dists[dists < gsim.minimum_distance] = gsim.minimum_distance
    # compute epsilon bin contributions only once
    eps_bands = truncnorm.cdf(epsilons[1:]) - truncnorm.cdf(epsilons[:-1])
    for rec in rupdata:
        rctx = contexts.RuptureContext(rec)
        dctx = contexts.DistancesContext(
            (param, rec[param][[sid]])
            for param in cmaker.REQUIRES_DISTANCES).roundup(
                gsim.minimum_distance)
        for m, imt in enumerate(iml2.imts):
            for p, poe in enumerate(iml2.poes_disagg):
                iml = iml2[m, p]
                with pne_mon:
                    pne = disaggregate_pne(gsim, rctx, sitecol, dctx, imt, iml,
                                           truncnorm, epsilons, eps_bands)
                acc[poe, str(imt), iml2.rlzi].append(pne)
    return acc
Ejemplo n.º 31
0
 def get_calc(self, testfile, job_ini, **kw):
     """
     Return the outputs of the calculation as a dictionary
     """
     self.testdir = os.path.dirname(testfile) if os.path.isfile(testfile) \
         else testfile
     inis = [os.path.join(self.testdir, ini) for ini in job_ini.split(',')]
     params = readinput.get_params(inis)
     params.update(kw)
     oq = oqvalidation.OqParam(**params)
     oq.validate()
     # change this when debugging the test
     monitor = Monitor(self.testdir)
     return base.calculators(oq, monitor)
Ejemplo n.º 32
0
class BaseCalculator(metaclass=abc.ABCMeta):
    """
    Abstract base class for all calculators.

    :param oqparam: OqParam object
    :param monitor: monitor object
    :param calc_id: numeric calculation ID
    """
    precalc = None
    accept_precalc = []
    from_engine = False  # set by engine.run_calc
    is_stochastic = False  # True for scenario and event based calculators

    def __init__(self, oqparam, calc_id=None):
        self.datastore = datastore.DataStore(calc_id)
        self._monitor = Monitor(
            '%s.run' % self.__class__.__name__, measuremem=True)
        self.oqparam = oqparam
        if 'performance_data' not in self.datastore:
            self.datastore.create_dset('performance_data', perf_dt)

    def monitor(self, operation='', **kw):
        """
        :returns: a new Monitor instance
        """
        mon = self._monitor(operation, hdf5=self.datastore.hdf5)
        self._monitor.calc_id = mon.calc_id = self.datastore.calc_id
        vars(mon).update(kw)
        return mon

    def save_params(self, **kw):
        """
        Update the current calculation parameters and save engine_version
        """
        if ('hazard_calculation_id' in kw and
                kw['hazard_calculation_id'] is None):
            del kw['hazard_calculation_id']
        vars(self.oqparam).update(**kw)
        self.datastore['oqparam'] = self.oqparam  # save the updated oqparam
        attrs = self.datastore['/'].attrs
        attrs['engine_version'] = engine_version
        attrs['date'] = datetime.now().isoformat()[:19]
        if 'checksum32' not in attrs:
            attrs['checksum32'] = readinput.get_checksum32(self.oqparam)
        self.datastore.flush()

    def check_precalc(self, precalc_mode):
        """
        Defensive programming against users providing an incorrect
        pre-calculation ID (with ``--hazard-calculation-id``).

        :param precalc_mode:
            calculation_mode of the previous calculation
        """
        calc_mode = self.oqparam.calculation_mode
        ok_mode = self.accept_precalc
        if calc_mode != precalc_mode and precalc_mode not in ok_mode:
            raise InvalidCalculationID(
                'In order to run a calculation of kind %r, '
                'you need to provide a calculation of kind %r, '
                'but you provided a %r instead' %
                (calc_mode, ok_mode, precalc_mode))

    def run(self, pre_execute=True, concurrent_tasks=None, close=True, **kw):
        """
        Run the calculation and return the exported outputs.
        """
        with self._monitor:
            self._monitor.username = kw.get('username', '')
            self._monitor.hdf5 = self.datastore.hdf5
            if concurrent_tasks is None:  # use the job.ini parameter
                ct = self.oqparam.concurrent_tasks
            else:  # used the parameter passed in the command-line
                ct = concurrent_tasks
            if ct == 0:  # disable distribution temporarily
                oq_distribute = os.environ.get('OQ_DISTRIBUTE')
                os.environ['OQ_DISTRIBUTE'] = 'no'
            if ct != self.oqparam.concurrent_tasks:
                # save the used concurrent_tasks
                self.oqparam.concurrent_tasks = ct
            self.save_params(**kw)
            try:
                if pre_execute:
                    self.pre_execute()
                self.result = self.execute()
                if self.result is not None:
                    self.post_execute(self.result)
                self.before_export()
                self.export(kw.get('exports', ''))
            except Exception:
                if kw.get('pdb'):  # post-mortem debug
                    tb = sys.exc_info()[2]
                    traceback.print_tb(tb)
                    pdb.post_mortem(tb)
                else:
                    logging.critical('', exc_info=True)
                    raise
            finally:
                # cleanup globals
                if ct == 0:  # restore OQ_DISTRIBUTE
                    if oq_distribute is None:  # was not set
                        del os.environ['OQ_DISTRIBUTE']
                    else:
                        os.environ['OQ_DISTRIBUTE'] = oq_distribute
                readinput.pmap = None
                readinput.exposure = None
                readinput.gmfs = None
                readinput.eids = None
                self._monitor.flush()

                if close:  # in the engine we close later
                    self.result = None
                    try:
                        self.datastore.close()
                    except (RuntimeError, ValueError):
                        # sometimes produces errors but they are difficult to
                        # reproduce
                        logging.warning('', exc_info=True)

        return getattr(self, 'exported', {})

    def core_task(*args):
        """
        Core routine running on the workers.
        """
        raise NotImplementedError

    @abc.abstractmethod
    def pre_execute(self):
        """
        Initialization phase.
        """

    @abc.abstractmethod
    def execute(self):
        """
        Execution phase. Usually will run in parallel the core
        function and return a dictionary with the results.
        """

    @abc.abstractmethod
    def post_execute(self, result):
        """
        Post-processing phase of the aggregated output. It must be
        overridden with the export code. It will return a dictionary
        of output files.
        """

    def export(self, exports=None):
        """
        Export all the outputs in the datastore in the given export formats.
        Individual outputs are not exported if there are multiple realizations.
        """
        self.exported = getattr(self.precalc, 'exported', {})
        if isinstance(exports, tuple):
            fmts = exports
        elif exports:  # is a string
            fmts = exports.split(',')
        elif isinstance(self.oqparam.exports, tuple):
            fmts = self.oqparam.exports
        else:  # is a string
            fmts = self.oqparam.exports.split(',')
        keys = set(self.datastore)
        has_hcurves = ('hcurves-stats' in self.datastore or
                       'hcurves-rlzs' in self.datastore)
        if has_hcurves:
            keys.add('hcurves')
        for fmt in fmts:
            if not fmt:
                continue
            for key in sorted(keys):  # top level keys
                if 'rlzs' in key and self.R > 1:
                    continue  # skip individual curves
                self._export((key, fmt))
            if has_hcurves and self.oqparam.hazard_maps:
                self._export(('hmaps', fmt))
            if has_hcurves and self.oqparam.uniform_hazard_spectra:
                self._export(('uhs', fmt))

    def _export(self, ekey):
        if ekey not in exp or self.exported.get(ekey):  # already exported
            return
        with self.monitor('export'):
            try:
                self.exported[ekey] = fnames = exp(ekey, self.datastore)
            except Exception as exc:
                fnames = []
                logging.error('Could not export %s: %s', ekey, exc)
            if fnames:
                logging.info('exported %s: %s', ekey[0], fnames)

    def before_export(self):
        """
        Set the attributes nbytes
        """
        # sanity check that eff_ruptures have been set, i.e. are not -1
        try:
            csm_info = self.datastore['csm_info']
        except KeyError:
            csm_info = self.datastore['csm_info'] = self.csm.info
        for sm in csm_info.source_models:
            for sg in sm.src_groups:
                assert sg.eff_ruptures != -1, sg

        for key in self.datastore:
            self.datastore.set_nbytes(key)
        self.datastore.flush()