def execute(self): self.datastore.flush() # just to be sure oq = self.oqparam self.set_param(hdf5path=self.datastore.filename, tempname=cache_epsilons(self.datastore, oq, self.assetcol, self.crmodel, self.E)) srcfilter = self.src_filter() logging.info('Sending {:_d} ruptures'.format( len(self.datastore['ruptures']))) self.events_per_sid = [] self.numlosses = 0 self.datastore.swmr_on() self.indices = general.AccumDict(accum=[]) # rlzi -> [(start, stop)] smap = parallel.Starmap(start_ebrisk, h5=self.datastore.hdf5) smap.monitor.save('srcfilter', srcfilter) smap.monitor.save('crmodel', self.crmodel) for rg in getters.gen_rupture_getters(self.datastore, oq.concurrent_tasks): smap.submit((rg, self.param)) smap.reduce(self.agg_dicts) if self.indices: self.datastore['event_loss_table/indices'] = self.indices gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum() logging.info('Produced %s of GMFs', general.humansize(gmf_bytes)) logging.info('Considered {:_d} / {:_d} losses'.format(*self.numlosses)) return 1
def pre_execute(self): """ Compute the GMFs, build the epsilons, the riskinputs, and a dictionary with the unit of measure, used in the export phase. """ oq = self.oqparam super().pre_execute() self.assetcol = self.datastore['assetcol'] self.event_slice = functools.partial(_event_slice, oq.number_of_ground_motion_fields) E = oq.number_of_ground_motion_fields * self.R self.riskinputs = self.build_riskinputs('gmf') self.param['tempname'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.crmodel, E) self.param['E'] = E # assuming the weights are the same for all IMTs try: self.param['weights'] = self.datastore['weights'][()] except KeyError: self.param['weights'] = [1 / self.R for _ in range(self.R)] self.param['event_slice'] = self.event_slice self.param['ael_dt'] = dt = ael_dt(oq.loss_names) A = len(self.assetcol) self.datastore.create_dset('loss_data/data', dt) self.datastore.create_dset('loss_data/indices', U32, (A, 2)) self.start = 0
def execute(self): self.datastore.flush() # just to be sure oq = self.oqparam parent = self.datastore.parent csm_info = parent['csm_info'] if parent else self.csm_info self.init_logic_tree(csm_info) self.set_param(hdf5path=self.datastore.filename, tempname=cache_epsilons(self.datastore, oq, self.assetcol, self.crmodel, self.E)) srcfilter = self.src_filter(self.datastore.tempname) logging.info('Sending %d ruptures', len(self.datastore['ruptures'])) self.events_per_sid = [] self.numlosses = 0 self.datastore.swmr_on() self.indices = general.AccumDict(accum=[]) # rlzi -> [(start, stop)] self.offset = 0 smap = parallel.Starmap(self.core_task.__func__, h5=self.datastore.hdf5) for rgetter in getters.gen_rupture_getters(self.datastore, srcfilter=srcfilter): smap.submit((rgetter, srcfilter, self.param)) smap.reduce(self.agg_dicts) if self.indices: self.datastore['asset_loss_table/indices'] = self.indices gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum() logging.info('Produced %s of GMFs', general.humansize(gmf_bytes)) logging.info('Stored {:_d} / {:_d} losses'.format(*self.numlosses)) return 1
def execute(self): self.datastore.flush() # just to be sure oq = self.oqparam parent = self.datastore.parent csm_info = parent['csm_info'] if parent else self.csm_info self.init_logic_tree(csm_info) self.set_param( hdf5path=self.datastore.filename, tempname=cache_epsilons( self.datastore, oq, self.assetcol, self.crmodel, self.E)) srcfilter = self.src_filter(self.datastore.tempname) maxw = self.E / (oq.concurrent_tasks or 1) logging.info('Reading %d ruptures', len(self.datastore['ruptures'])) allargs = ((rgetter, srcfilter, self.param) for rgetter in getters.gen_rupture_getters( self.datastore, maxweight=maxw)) self.events_per_sid = [] self.lossbytes = 0 self.datastore.swmr_on() smap = parallel.Starmap( self.core_task.__func__, allargs, h5=self.datastore.hdf5) smap.reduce(self.agg_dicts) gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum() logging.info( 'Produced %s of GMFs', general.humansize(gmf_bytes)) logging.info( 'Produced %s of losses', general.humansize(self.lossbytes)) return 1
def execute(self): self.datastore.flush() # just to be sure oq = self.oqparam self.set_param(hdf5path=self.datastore.filename, tempname=cache_epsilons(self.datastore, oq, self.assetcol, self.crmodel, self.E)) srcfilter = self.src_filter() logging.info('Sending {:_d} ruptures'.format( len(self.datastore['ruptures']))) self.events_per_sid = [] self.datastore.swmr_on() self.avg_gmf = general.AccumDict(accum=numpy.zeros(self.N, F32)) # imt -> gmvs smap = parallel.Starmap(start_ebrisk, h5=self.datastore.hdf5) smap.monitor.save('srcfilter', srcfilter) smap.monitor.save('crmodel', self.crmodel) for rg in getters.gen_rupture_getters(self.datastore, oq.concurrent_tasks): smap.submit((rg, self.param)) smap.reduce(self.agg_dicts) gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum() logging.info('Produced %s of GMFs', general.humansize(gmf_bytes)) size = general.humansize(self.datastore.getsize('agg_loss_table')) logging.info('Stored %s in the agg_loss_table', size) return 1
def pre_execute(self): oq = self.oqparam super().pre_execute() parent = self.datastore.parent if not oq.ground_motion_fields: return # this happens in the reportwriter self.L = len(self.crmodel.lti) self.T = len(self.assetcol.tagcol) self.A = len(self.assetcol) if parent: self.datastore['csm_info'] = parent['csm_info'] self.events = parent['events'][('id', 'rlz_id')] logging.info('There are %d ruptures and %d events', len(parent['ruptures']), len(self.events)) else: self.events = self.datastore['events'][('id', 'rlz_id')] if oq.return_periods != [0]: # setting return_periods = 0 disable loss curves and maps eff_time = oq.investigation_time * oq.ses_per_logic_tree_path if eff_time < 2: logging.warning( 'eff_time=%s is too small to compute loss curves', eff_time) else: self.param['builder'] = get_loss_builder( parent if parent else self.datastore, oq.return_periods, oq.loss_dt()) # sorting the eids is essential to get the epsilons in the right # order (i.e. consistent with the one used in ebr from ruptures) self.riskinputs = self.build_riskinputs('gmf') self.param['tempname'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.crmodel, self.E) self.param['avg_losses'] = oq.avg_losses self.param['ses_ratio'] = oq.ses_ratio self.param['stats'] = list(oq.hazard_stats().items()) self.param['conditional_loss_poes'] = oq.conditional_loss_poes self.taskno = 0 self.start = 0 avg_losses = oq.avg_losses if avg_losses: self.dset = self.datastore.create_dset('avg_losses-rlzs', F32, (self.A, self.R, self.L)) self.agglosses = numpy.zeros((self.E, self.L), F32) if 'builder' in self.param: logging.warning( 'Building the loss curves and maps for each asset is ' 'deprecated: consider building the aggregate curves and ' 'maps with the ebrisk calculator instead') self.build_datasets(self.param['builder']) if parent: parent.close() # avoid concurrent reading issues
def pre_execute(self): oq = self.oqparam super().pre_execute() parent = self.datastore.parent if not oq.ground_motion_fields: return # this happens in the reportwriter self.L = len(self.riskmodel.lti) self.T = len(self.assetcol.tagcol) self.A = len(self.assetcol) if parent: self.datastore['csm_info'] = parent['csm_info'] self.events = parent['events'][('id', 'rlz')] else: self.events = self.datastore['events'][('id', 'rlz')] if oq.return_periods != [0]: # setting return_periods = 0 disable loss curves and maps eff_time = oq.investigation_time * oq.ses_per_logic_tree_path if eff_time < 2: logging.warning( 'eff_time=%s is too small to compute loss curves', eff_time) else: self.param['builder'] = get_loss_builder( parent if parent else self.datastore, oq.return_periods, oq.loss_dt()) # sorting the eids is essential to get the epsilons in the right # order (i.e. consistent with the one used in ebr from ruptures) self.riskinputs = self.build_riskinputs('gmf') self.param['epspath'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.riskmodel, self.E) self.param['avg_losses'] = oq.avg_losses self.param['ses_ratio'] = oq.ses_ratio self.param['stats'] = list(oq.hazard_stats().items()) self.param['conditional_loss_poes'] = oq.conditional_loss_poes self.taskno = 0 self.start = 0 avg_losses = oq.avg_losses if avg_losses: self.dset = self.datastore.create_dset( 'avg_losses-rlzs', F32, (self.A, self.R, self.L)) self.agglosses = numpy.zeros((self.E, self.L), F32) if 'builder' in self.param: logging.warning( 'Building the loss curves and maps for each asset is ' 'deprecated: consider building the aggregate curves and ' 'maps with the ebrisk calculator instead') self.build_datasets(self.param['builder']) if parent: parent.close() # avoid concurrent reading issues
def execute(self): oq = self.oqparam self.set_param( num_taxonomies=self.assetcol.num_taxonomies_by_site(), maxweight=oq.ebrisk_maxweight / (oq.concurrent_tasks or 1), epspath=cache_epsilons( self.datastore, oq, self.assetcol, self.riskmodel, self.E)) parent = self.datastore.parent if parent: hdf5path = parent.filename grp_indices = parent['ruptures'].attrs['grp_indices'] nruptures = len(parent['ruptures']) else: hdf5path = self.datastore.hdf5cache() grp_indices = self.datastore['ruptures'].attrs['grp_indices'] nruptures = len(self.datastore['ruptures']) with hdf5.File(hdf5path, 'r+') as cache: self.datastore.hdf5.copy('weights', cache) self.datastore.hdf5.copy('ruptures', cache) self.datastore.hdf5.copy('rupgeoms', cache) self.init_logic_tree(self.csm_info) smap = parallel.Starmap( self.core_task.__func__, monitor=self.monitor()) trt_by_grp = self.csm_info.grp_by("trt") samples = self.csm_info.get_samples_by_grp() rlzs_by_gsim_grp = self.csm_info.get_rlzs_by_gsim_grp() ruptures_per_block = numpy.ceil(nruptures / (oq.concurrent_tasks or 1)) first_event = 0 for grp_id, rlzs_by_gsim in rlzs_by_gsim_grp.items(): start, stop = grp_indices[grp_id] for indices in general.block_splitter( range(start, stop), ruptures_per_block): rgetter = getters.RuptureGetter( hdf5path, list(indices), grp_id, trt_by_grp[grp_id], samples[grp_id], rlzs_by_gsim, first_event) first_event += rgetter.num_events smap.submit(rgetter, self.src_filter, self.param) self.events_per_sid = [] self.gmf_nbytes = 0 res = smap.reduce(self.agg_dicts, numpy.zeros(self.N)) logging.info('Produced %s of GMFs', general.humansize(self.gmf_nbytes)) return res
def execute(self): oq = self.oqparam self.set_param( num_taxonomies=self.assetcol.num_taxonomies_by_site(), maxweight=oq.ebrisk_maxweight / (oq.concurrent_tasks or 1), epspath=cache_epsilons(self.datastore, oq, self.assetcol, self.riskmodel, self.E)) parent = self.datastore.parent if parent: hdf5path = parent.filename grp_indices = parent['ruptures'].attrs['grp_indices'] nruptures = len(parent['ruptures']) else: hdf5path = self.datastore.hdf5cache() grp_indices = self.datastore['ruptures'].attrs['grp_indices'] nruptures = len(self.datastore['ruptures']) with hdf5.File(hdf5path, 'r+') as cache: self.datastore.hdf5.copy('weights', cache) self.datastore.hdf5.copy('ruptures', cache) self.datastore.hdf5.copy('rupgeoms', cache) self.init_logic_tree(self.csm_info) smap = parallel.Starmap(self.core_task.__func__, monitor=self.monitor()) trt_by_grp = self.csm_info.grp_by("trt") samples = self.csm_info.get_samples_by_grp() rlzs_by_gsim_grp = self.csm_info.get_rlzs_by_gsim_grp() ruptures_per_block = numpy.ceil(nruptures / (oq.concurrent_tasks or 1)) first_event = 0 for grp_id, rlzs_by_gsim in rlzs_by_gsim_grp.items(): start, stop = grp_indices[grp_id] for indices in general.block_splitter(range(start, stop), ruptures_per_block): rgetter = getters.RuptureGetter(hdf5path, list(indices), grp_id, trt_by_grp[grp_id], samples[grp_id], rlzs_by_gsim, first_event) first_event += rgetter.num_events smap.submit(rgetter, self.src_filter, self.param) self.events_per_sid = [] self.gmf_nbytes = 0 res = smap.reduce(self.agg_dicts, numpy.zeros(self.N)) logging.info('Produced %s of GMFs', general.humansize(self.gmf_nbytes)) return res
def pre_execute(self): """ Compute the GMFs, build the epsilons, the riskinputs, and a dictionary with the unit of measure, used in the export phase. """ oq = self.oqparam if not oq.ground_motion_fields: return # this happens in the reportwriter parent = self.datastore.parent if parent: self.datastore['full_lt'] = parent['full_lt'] ne = len(parent['events']) logging.info('There are %d ruptures and %d events', len(parent['ruptures']), ne) if oq.investigation_time and oq.return_periods != [0]: # setting return_periods = 0 disable loss curves eff_time = oq.investigation_time * oq.ses_per_logic_tree_path if eff_time < 2: logging.warning( 'eff_time=%s is too small to compute loss curves', eff_time) super().pre_execute() self.assetcol = self.datastore['assetcol'] self.riskinputs = self.build_riskinputs('gmf') self.param['tempname'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.crmodel, self.E) self.param['aggregate_by'] = oq.aggregate_by self.param['secondary_simulations'] = oq.secondary_simulations self.param['master_seed'] = oq.master_seed self.rlzs = self.datastore['events']['rlz_id'] self.num_events = numpy.bincount(self.rlzs) # events by rlz aggkey = self.assetcol.tagcol.get_aggkey(oq.aggregate_by) self.param['alt'] = self.acc = scientific.AggLossTable.new( aggkey, oq.loss_names, sec_losses=[]) L = len(oq.loss_names) self.avglosses = numpy.zeros((len(self.assetcol), self.R, L), F32) if oq.investigation_time: # event_based self.avg_ratio = numpy.array([oq.ses_ratio] * self.R) else: # scenario self.avg_ratio = 1. / self.num_events
def pre_execute(self): """ Compute the GMFs, build the epsilons, the riskinputs, and a dictionary with the unit of measure, used in the export phase. """ oq = self.oqparam super().pre_execute() self.assetcol = self.datastore['assetcol'] E = oq.number_of_ground_motion_fields * self.R self.riskinputs = self.build_riskinputs('gmf') self.param['tempname'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.crmodel, E) self.param['E'] = E # assuming the weights are the same for all IMTs try: self.param['weights'] = self.datastore['weights'][()] except KeyError: self.param['weights'] = [1 / self.R for _ in range(self.R)] self.rlzs = self.datastore['events']['rlz_id'] self.param['num_events'] = numpy.bincount(self.rlzs) # events by rlz
def pre_execute(self): """ Compute the GMFs, build the epsilons, the riskinputs, and a dictionary with the unit of measure, used in the export phase. """ oq = self.oqparam super().pre_execute() self.assetcol = self.datastore['assetcol'] self.event_slice = functools.partial( _event_slice, oq.number_of_ground_motion_fields) E = oq.number_of_ground_motion_fields * self.R self.riskinputs = self.build_riskinputs('gmf') self.param['epspath'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.riskmodel, E) self.param['E'] = E # assuming the weights are the same for all IMTs try: self.param['weights'] = self.datastore['weights'][()] except KeyError: self.param['weights'] = [1 / self.R for _ in range(self.R)] self.param['event_slice'] = self.event_slice self.param['asset_loss_table'] = self.oqparam.asset_loss_table
def pre_execute(self): """ Compute the GMFs, build the epsilons, the riskinputs, and a dictionary with the unit of measure, used in the export phase. """ oq = self.oqparam super().pre_execute() self.assetcol = self.datastore['assetcol'] self.event_slice = functools.partial(_event_slice, oq.number_of_ground_motion_fields) E = oq.number_of_ground_motion_fields * self.R self.riskinputs = self.build_riskinputs('gmf') self.param['epspath'] = riskinput.cache_epsilons( self.datastore, oq, self.assetcol, self.crmodel, E) self.param['E'] = E # assuming the weights are the same for all IMTs try: self.param['weights'] = self.datastore['weights'][()] except KeyError: self.param['weights'] = [1 / self.R for _ in range(self.R)] self.param['event_slice'] = self.event_slice self.param['asset_loss_table'] = self.oqparam.asset_loss_table
def execute(self): self.datastore.flush() # just to be sure oq = self.oqparam parent = self.datastore.parent if parent: grp_indices = parent['ruptures'].attrs['grp_indices'] n_occ = parent['ruptures']['n_occ'] dstore = parent csm_info = parent['csm_info'] else: grp_indices = self.datastore['ruptures'].attrs['grp_indices'] n_occ = self.datastore['ruptures']['n_occ'] dstore = self.datastore csm_info = self.csm_info per_block = numpy.ceil(n_occ.sum() / (oq.concurrent_tasks or 1)) self.set_param( hdf5path=self.datastore.filename, task_duration=oq.task_duration or 1200, # 20min tempname=cache_epsilons(self.datastore, oq, self.assetcol, self.crmodel, self.E)) self.init_logic_tree(csm_info) trt_by_grp = csm_info.grp_by("trt") samples = csm_info.get_samples_by_grp() rlzs_by_gsim_grp = csm_info.get_rlzs_by_gsim_grp() ngroups = 0 fe = 0 eslices = self.datastore['eslices'] allargs = [] allpairs = list(enumerate(n_occ)) srcfilter = self.src_filter(self.datastore.tempname) for grp_id, rlzs_by_gsim in rlzs_by_gsim_grp.items(): start, stop = grp_indices[grp_id] if start == stop: # no ruptures for the given grp_id continue ngroups += 1 for pairs in general.block_splitter(allpairs[start:stop], per_block, weight=get_n_occ): indices = [i for i, n in pairs] rup_array = dstore['ruptures'][indices] rgetter = getters.RuptureGetter( rup_array, dstore.filename, grp_id, trt_by_grp[grp_id], samples[grp_id], rlzs_by_gsim, eslices[fe:fe + len(indices), 0]) allargs.append((rgetter, srcfilter, self.param)) fe += len(indices) logging.info('Found %d/%d source groups with ruptures', ngroups, len(rlzs_by_gsim_grp)) self.events_per_sid = [] self.lossbytes = 0 self.datastore.swmr_on() smap = parallel.Starmap(self.core_task.__func__, allargs, h5=self.datastore.hdf5) res = smap.reduce(self.agg_dicts, numpy.zeros(self.N)) gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum() logging.info('Produced %s of GMFs', general.humansize(gmf_bytes)) logging.info('Produced %s of losses', general.humansize(self.lossbytes)) return res
def execute(self): oq = self.oqparam parent = self.datastore.parent if parent: hdf5path = parent.filename grp_indices = parent['ruptures'].attrs['grp_indices'] n_occ = parent['ruptures']['n_occ'] else: hdf5path = self.datastore.cachepath() grp_indices = self.datastore['ruptures'].attrs['grp_indices'] n_occ = self.datastore['ruptures']['n_occ'] with hdf5.File(hdf5path, 'r+') as cache: self.datastore.hdf5.copy('weights', cache) self.datastore.hdf5.copy('ruptures', cache) self.datastore.hdf5.copy('rupgeoms', cache) num_cores = oq.__class__.concurrent_tasks.default // 2 or 1 per_block = numpy.ceil(n_occ.sum() / (oq.concurrent_tasks or 1)) logging.info( 'Using %d occurrences per block (over %d occurrences, ' '%d events)', per_block, n_occ.sum(), self.E) self.set_param( task_duration=oq.task_duration or 600, # 10min epspath=cache_epsilons(self.datastore, oq, self.assetcol, self.crmodel, self.E)) self.init_logic_tree(self.csm_info) trt_by_grp = self.csm_info.grp_by("trt") samples = self.csm_info.get_samples_by_grp() rlzs_by_gsim_grp = self.csm_info.get_rlzs_by_gsim_grp() ngroups = 0 fe = 0 eslices = self.datastore['eslices'] allargs = [] allpairs = list(enumerate(n_occ)) for grp_id, rlzs_by_gsim in rlzs_by_gsim_grp.items(): start, stop = grp_indices[grp_id] if start == stop: # no ruptures for the given grp_id continue ngroups += 1 for pairs in general.block_splitter(allpairs[start:stop], per_block, weight=get_n_occ): indices = [i for i, n in pairs] rgetter = getters.RuptureGetter( hdf5path, indices, grp_id, trt_by_grp[grp_id], samples[grp_id], rlzs_by_gsim, eslices[fe:fe + len(indices), 0]) allargs.append((rgetter, self.src_filter, self.param)) fe += len(indices) logging.info('Found %d/%d source groups with ruptures', ngroups, len(rlzs_by_gsim_grp)) self.events_per_sid = [] self.lossbytes = 0 smap = parallel.Starmap(self.core_task.__func__, allargs, num_cores=num_cores, hdf5path=self.datastore.filename) res = smap.reduce(self.agg_dicts, numpy.zeros(self.N)) gmf_bytes = self.datastore['gmf_info']['gmfbytes'].sum() self.datastore.set_attrs('gmf_info', events_per_sid=self.events_per_sid) logging.info('Produced %s of GMFs', general.humansize(gmf_bytes)) logging.info('Produced %s of losses', general.humansize(self.lossbytes)) return res