def pmap_from_grp( sources, sites, imtls, gsims, truncation_level=None, source_site_filter='SourceSitesFilter', maximum_distance=None, bbs=(), monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a ProbabilityMap instance """ if source_site_filter == 'SourceSitesFilter': # default source_site_filter = ( filters.SourceSitesFilter(maximum_distance) if maximum_distance else filters.source_site_noop_filter) with GroundShakingIntensityModel.forbid_instantiation(): imtls = DictArray(imtls) cmaker = ContextMaker(gsims, maximum_distance) ctx_mon = monitor('making contexts', measuremem=False) pne_mon = monitor('computing poes', measuremem=False) disagg_mon = monitor('get closest points', measuremem=False) monitor.calc_times = [] # pairs (src_id, delta_t) pmap = ProbabilityMap(len(imtls.array), len(gsims)) for src, s_sites in source_site_filter(sources, sites): t0 = time.time() pmap |= poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs, ctx_mon, pne_mon, disagg_mon) # we are attaching the calculation times to the monitor # so that oq-lite (and the engine) can store them monitor.calc_times.append((src.id, time.time() - t0)) # NB: source.id is an integer; it should not be confused # with source.source_id, which is a string monitor.eff_ruptures = pne_mon.counts # contributing ruptures return pmap
def wrapped(*args): """ Initialize logs, make sure the job is still running, and run the task code surrounded by a try-except. If any error occurs, log it as a critical failure. """ # the last argument is assumed to be a monitor monitor = args[-1] job = models.OqJob.objects.get(id=monitor.job_id) if job.is_running is False: # the job was killed, it is useless to run the task raise JobNotRunning(monitor.job_id) # it is important to save the task id soon, so that # the revoke functionality can work with monitor("storing task id", task=tsk, autoflush=True): pass with logs.handle(job): check_mem_usage() # warn if too much memory is used # run the task try: total = "total " + task_func.__name__ with monitor(total, task=tsk): with GroundShakingIntensityModel.forbid_instantiation(): return task_func(*args) finally: # save on the db CacheInserter.flushall() # the task finished, we can remove from the performance # table the associated row 'storing task id' models.Performance.objects.filter( oq_job=job, operation="storing task id", task_id=tsk.request.id ).delete()
def classical(group, src_filter, gsims, param, monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times, .eff_ruptures """ if getattr(group, 'src_interdep', None) == 'mutex': mutex_weight = { src.source_id: weight for src, weight in zip(group.sources, group.srcs_weights) } else: mutex_weight = None grp_ids = set() for src in group: grp_ids.update(src.src_group_ids) maxdist = src_filter.integration_distance with GroundShakingIntensityModel.forbid_instantiation(): imtls = param['imtls'] trunclevel = param.get('truncation_level') cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('make_contexts', measuremem=False) poe_mon = monitor('get_poes', measuremem=False) pmap = AccumDict({ grp_id: ProbabilityMap(len(imtls.array), len(gsims)) for grp_id in grp_ids }) # AccumDict of arrays with 4 elements weight, nsites, calc_time, split pmap.calc_times = AccumDict(accum=numpy.zeros(4)) pmap.eff_ruptures = AccumDict() # grp_id -> num_ruptures for src, s_sites in src_filter(group): # filter now t0 = time.time() indep = group.rup_interdep == 'indep' if mutex_weight else True poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon, poe_mon, indep) if mutex_weight: # mutex sources weight = mutex_weight[src.source_id] for sid in poemap: pcurve = pmap[group.id].setdefault(sid, 0) pcurve += poemap[sid] * weight elif poemap: for grp_id in src.src_group_ids: pmap[grp_id] |= poemap src_id = src.source_id.split(':', 1)[0] pmap.calc_times[src_id] += numpy.array( [src.weight, len(s_sites), time.time() - t0, 1]) # storing the number of contributing ruptures too pmap.eff_ruptures += { grp_id: getattr(poemap, 'eff_ruptures', 0) for grp_id in src.src_group_ids } if mutex_weight and group.grp_probability is not None: pmap[group.id] *= group.grp_probability return pmap
def wrapper(*args): check_mem_usage() # check if too much memory is used monitor = args[-1] monitor.flush = NoFlush(monitor, func.__name__) with monitor('total ' + func.__name__, measuremem=True), \ GroundShakingIntensityModel.forbid_instantiation(): result = func(*args) rec_delattr(monitor, 'flush') return result
def wrapper(*args): monitor = args[-1] check_mem_usage(monitor) # check if too much memory is used monitor.flush = NoFlush(monitor, func.__name__) with monitor('total ' + func.__name__, measuremem=True), \ GroundShakingIntensityModel.forbid_instantiation(): result = func(*args) rec_delattr(monitor, 'flush') return result
def wrapper(*args): monitor = args[-1] check_mem_usage(monitor) # check if too much memory is used monitor.flush = NoFlush(monitor, func.__name__) with monitor('total ' + func.__name__, measuremem=True), \ GroundShakingIntensityModel.forbid_instantiation(): result = func(*args) # NB: flush must not be called in the workers - they must not # have access to the datastore - so we remove it rec_delattr(monitor, 'flush') return result
def pmap_from_grp(sources, source_site_filter, imtls, gsims, truncation_level=None, bbs=(), monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a ProbabilityMap instance """ if isinstance(sources, SourceGroup): group = sources sources = group.sources trt = sources[0].tectonic_region_type else: # list of sources trt = sources[0].tectonic_region_type group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep') try: maxdist = source_site_filter.integration_distance[trt] except: maxdist = source_site_filter.integration_distance if hasattr(gsims, 'keys'): # dictionary trt -> gsim gsims = [gsims[trt]] with GroundShakingIntensityModel.forbid_instantiation(): imtls = DictArray(imtls) cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('making contexts', measuremem=False) pne_mon = monitor('computing poes', measuremem=False) disagg_mon = monitor('get closest points', measuremem=False) src_indep = group.src_interdep == 'indep' pmap = ProbabilityMap(len(imtls.array), len(gsims)) pmap.calc_times = [] # pairs (src_id, delta_t) pmap.grp_id = sources[0].src_group_id for src, s_sites in source_site_filter(sources): t0 = time.time() poemap = poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs, group.rup_interdep == 'indep', ctx_mon, pne_mon, disagg_mon) if src_indep: # usual composition of probabilities pmap |= poemap else: # mutually exclusive probabilities weight = float(group.srcs_weights[src.source_id]) for sid in poemap: pmap[sid] += poemap[sid] * weight pmap.calc_times.append( (src.source_id, len(s_sites), time.time() - t0)) # storing the number of contributing ruptures too pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts} return pmap
def pmap_from_grp( sources, source_site_filter, imtls, gsims, truncation_level=None, bbs=(), monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a ProbabilityMap instance """ if isinstance(sources, SourceGroup): group = sources sources = group.src_list else: group = SourceGroup(sources, 'src_group', 'indep', 'indep') sources = group.src_list trt = sources[0].tectonic_region_type try: maxdist = source_site_filter.integration_distance[trt] except: maxdist = source_site_filter.integration_distance if hasattr(gsims, 'keys'): gsims = [gsims[trt]] # check all the sources belong to the same tectonic region trts = set(src.tectonic_region_type for src in sources) assert len(trts) == 1, 'Multiple TRTs: %s' % ', '.join(trts) with GroundShakingIntensityModel.forbid_instantiation(): imtls = DictArray(imtls) cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('making contexts', measuremem=False) pne_mon = monitor('computing poes', measuremem=False) disagg_mon = monitor('get closest points', measuremem=False) src_indep = group.src_interdep == 'indep' pmap = ProbabilityMap(len(imtls.array), len(gsims)) pmap.calc_times = [] # pairs (src_id, delta_t) pmap.grp_id = sources[0].src_group_id for src, s_sites in source_site_filter(sources): t0 = time.time() poemap = poe_map( src, s_sites, imtls, cmaker, truncation_level, bbs, group.rup_interdep == 'indep', ctx_mon, pne_mon, disagg_mon) if src_indep: # usual composition of probabilities pmap |= poemap else: # mutually exclusive probabilities weight = float(group.srcs_weights[src.source_id]) for sid in poemap: pmap[sid] += poemap[sid] * weight pmap.calc_times.append( (src.source_id, len(s_sites), time.time() - t0)) # storing the number of contributing ruptures too pmap.eff_ruptures = {pmap.grp_id: pne_mon.counts} return pmap
def wrapper(*args): monitor = args[-1] check_mem_usage(monitor) # check if too much memory is used monitor.flush = NoFlush(monitor, func.__name__) with monitor('total ' + func.__name__, measuremem=True), \ GroundShakingIntensityModel.forbid_instantiation(): result = func(*args) # NB: flush must not be called in the workers - they must not # have access to the datastore - so we remove it rec_delattr(monitor, 'flush') return result
def pmap_from_trt(sources, src_filter, gsims, param, monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. :returns: a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times, .eff_ruptures """ maxdist = src_filter.integration_distance srcs = [] grp_ids = set() for src in sources: if hasattr(src, '__iter__'): # MultiPointSource srcs.extend(src) else: srcs.append(src) grp_ids.update(src.src_group_ids) del sources with GroundShakingIntensityModel.forbid_instantiation(): imtls = param['imtls'] trunclevel = param.get('truncation_level') cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('making contexts', measuremem=False) pne_mons = [ monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims ] pmap = AccumDict({ grp_id: ProbabilityMap(len(imtls.array), len(gsims)) for grp_id in grp_ids }) pmap.calc_times = [] # pairs (src_id, delta_t) pmap.eff_ruptures = AccumDict() # grp_id -> num_ruptures for src, s_sites in src_filter(srcs): t0 = time.time() poe = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon, pne_mons) for grp_id in src.src_group_ids: pmap[grp_id] |= poe pmap.calc_times.append( (src.source_id, src.weight, len(s_sites), time.time() - t0)) # storing the number of contributing ruptures too pmap.eff_ruptures += { grp_id: poe.eff_ruptures for grp_id in src.src_group_ids } return pmap
def pmap_from_trt(sources, src_filter, gsims, param, monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. :returns: a dictionary {grp_id: pmap} with attributes .grp_ids, .calc_times, .eff_ruptures """ grp_ids = set() for src in sources: grp_ids.update(src.src_group_ids) maxdist = src_filter.integration_distance srcs = sum([split_source(src) for src in sources], []) # split first with GroundShakingIntensityModel.forbid_instantiation(): imtls = param['imtls'] trunclevel = param.get('truncation_level') cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('make_contexts', measuremem=False) poe_mon = monitor('get_poes', measuremem=False) pmap = AccumDict({ grp_id: ProbabilityMap(len(imtls.array), len(gsims)) for grp_id in grp_ids }) pmap.calc_times = [] # pairs (src_id, delta_t) pmap.eff_ruptures = AccumDict() # grp_id -> num_ruptures for src, s_sites in src_filter(srcs): # filter now t0 = time.time() poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon, poe_mon) if poemap: for grp_id in src.src_group_ids: pmap[grp_id] |= poemap pmap.calc_times.append( (src.source_id, src.weight, len(s_sites), time.time() - t0)) # storing the number of contributing ruptures too pmap.eff_ruptures += { grp_id: getattr(poemap, 'eff_ruptures', 0) for grp_id in src.src_group_ids } return pmap
def pmap_from_grp(sources, sites, imtls, gsims, truncation_level=None, source_site_filter='SourceSitesFilter', maximum_distance=None, bbs=(), monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a ProbabilityMap instance """ if source_site_filter == 'SourceSitesFilter': # default source_site_filter = (filters.SourceSitesFilter(maximum_distance) if maximum_distance else filters.source_site_noop_filter) with GroundShakingIntensityModel.forbid_instantiation(): imtls = DictArray(imtls) cmaker = ContextMaker(gsims, maximum_distance) ctx_mon = monitor('making contexts', measuremem=False) pne_mon = monitor('computing poes', measuremem=False) disagg_mon = monitor('get closest points', measuremem=False) monitor.calc_times = [] # pairs (src_id, delta_t) pmap = ProbabilityMap(len(imtls.array), len(gsims)) for src, s_sites in source_site_filter(sources, sites): t0 = time.time() pmap |= poe_map(src, s_sites, imtls, cmaker, truncation_level, bbs, ctx_mon, pne_mon, disagg_mon) # we are attaching the calculation times to the monitor # so that oq-lite (and the engine) can store them monitor.calc_times.append( (src.source_id, len(s_sites), time.time() - t0)) # NB: source.id is an integer; it should not be confused # with source.source_id, which is a string monitor.eff_ruptures = pne_mon.counts # contributing ruptures return pmap
def pmap_from_grp(group, src_filter, gsims, param, monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a dictionary {grp_id: ProbabilityMap instance} """ mutex_weight = { src.source_id: weight for src, weight in zip(group.sources, group.srcs_weights) } maxdist = src_filter.integration_distance srcs = sum([split_source(src) for src in group.sources], []) with GroundShakingIntensityModel.forbid_instantiation(): imtls = param['imtls'] trunclevel = param.get('truncation_level') cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('make_contexts', measuremem=False) poe_mon = monitor('get_poes', measuremem=False) pmap = ProbabilityMap(len(imtls.array), len(gsims)) calc_times = [] # pairs (src_id, delta_t) for src, s_sites in src_filter(srcs): t0 = time.time() poemap = cmaker.poe_map(src, s_sites, imtls, trunclevel, ctx_mon, poe_mon, group.rup_interdep == 'indep') weight = mutex_weight[src.source_id] for sid in poemap: pcurve = pmap.setdefault(sid, 0) pcurve += poemap[sid] * weight calc_times.append( (src.source_id, src.weight, len(s_sites), time.time() - t0)) if group.grp_probability is not None: pmap *= group.grp_probability acc = AccumDict({group.id: pmap}) # adding the number of contributing ruptures too acc.eff_ruptures = {group.id: ctx_mon.counts} acc.calc_times = calc_times return acc
def pmap_from_grp(sources, src_filter, gsims, param, monitor=Monitor()): """ Compute the hazard curves for a set of sources belonging to the same tectonic region type for all the GSIMs associated to that TRT. The arguments are the same as in :func:`calc_hazard_curves`, except for ``gsims``, which is a list of GSIM instances. :returns: a ProbabilityMap instance """ if isinstance(sources, SourceGroup): group = sources sources = group.sources trt = sources[0].tectonic_region_type mutex_weight = { src.source_id: weight for src, weight in zip(group.sources, group.srcs_weights) } else: # list of sources trt = sources[0].tectonic_region_type group = SourceGroup(trt, sources, 'src_group', 'indep', 'indep') grp_id = sources[0].src_group_id maxdist = src_filter.integration_distance if hasattr(gsims, 'keys'): # dictionary trt -> gsim gsims = [gsims[trt]] srcs = [] for src in sources: if hasattr(src, '__iter__'): # MultiPointSource srcs.extend(src) else: srcs.append(src) del sources with GroundShakingIntensityModel.forbid_instantiation(): imtls = param['imtls'] trunclevel = param.get('truncation_level') cmaker = ContextMaker(gsims, maxdist) ctx_mon = monitor('making contexts', measuremem=False) pne_mons = [ monitor('%s.get_poes' % gsim, measuremem=False) for gsim in gsims ] src_indep = group.src_interdep == 'indep' pmap = ProbabilityMap(len(imtls.array), len(gsims)) pmap.calc_times = [] # pairs (src_id, delta_t) pmap.grp_id = grp_id for src, s_sites in src_filter(srcs): t0 = time.time() poemap = poe_map(src, s_sites, imtls, cmaker, trunclevel, ctx_mon, pne_mons, group.rup_interdep == 'indep') if src_indep: # usual composition of probabilities pmap |= poemap else: # mutually exclusive probabilities weight = mutex_weight[src.source_id] for sid in poemap: pcurve = pmap.setdefault(sid, 0) pcurve += poemap[sid] * weight pmap.calc_times.append( (src.source_id, src.weight, len(s_sites), time.time() - t0)) # storing the number of contributing ruptures too pmap.eff_ruptures = {pmap.grp_id: pne_mons[0].counts} if group.grp_probability is not None: return pmap * group.grp_probability return pmap