def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ job_info = hdf5.LiteralAttrs() if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator precalc_id = self.oqparam.hazard_calculation_id if precalc_id is None: # recompute everything self.compute_previous() else: # read previously computed data self.read_previous(precalc_id) self.init() else: # we are in a basic calculator self.basic_pre_execute() if 'source' in self.oqparam.inputs: vars(job_info).update(readinput.get_job_info( self.oqparam, self.csm, self.sitecol)) job_info.hostname = socket.gethostname() if hasattr(self, 'riskmodel'): job_info.require_epsilons = bool(self.riskmodel.covs) self.job_info = job_info self.datastore.flush() try: csm_info = self.datastore['csm_info'] except KeyError: pass else: csm_info.gsim_lt.check_imts(self.oqparam.imtls)
def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ precalc_id = self.oqparam.hazard_calculation_id job_info = {} if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator self.precalc = (self.compute_previous() if precalc_id is None else self.read_previous(precalc_id)) self.init() else: # we are in a basic calculator self.precalc = None self.basic_pre_execute() if 'source' in self.oqparam.inputs: job_info.update( readinput.get_job_info(self.oqparam, self.csm, self.sitecol)) if hasattr(self, 'riskmodel'): job_info['require_epsilons'] = bool(self.riskmodel.covs) self._monitor.save_info(job_info) self.param = {} # used in the risk calculators
def read_sources(self): """ Read the composite source model (if any). This method must be called after read_exposure_sitecol, to be able to filter to sources according to the site collection. """ if 'source' in self.oqparam.inputs: logging.info('Reading the composite source model') with self.monitor( 'reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model( self.oqparam, self.sitecol, self.SourceProcessor, self.monitor) # we could manage limits here self.source_pre_info = self.csm.source_info self.job_info = readinput.get_job_info( self.oqparam, self.csm, self.sitecol) self.csm.count_ruptures() self.rlzs_assoc = self.csm.get_rlzs_assoc() logging.info( 'Total weight of the sources=%s', self.job_info['input_weight']) logging.info( 'Expected output size=%s', self.job_info['output_weight'])
def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ job_info = hdf5.LiteralAttrs() if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator precalc_id = self.oqparam.hazard_calculation_id if precalc_id is None: # recompute everything self.compute_previous() else: # read previously computed data self.read_previous(precalc_id) self.init() else: # we are in a basic calculator self.basic_pre_execute() if 'source' in self.oqparam.inputs: vars(job_info).update( readinput.get_job_info(self.oqparam, self.csm, self.sitecol)) job_info.hostname = socket.gethostname() if hasattr(self, 'riskmodel'): job_info.require_epsilons = bool(self.riskmodel.covs) self.job_info = job_info self.datastore.flush() try: csm_info = self.datastore['csm_info'] except KeyError: pass else: csm_info.gsim_lt.check_imts(self.oqparam.imtls)
def read_sources(self): """ Read the composite source model (if any). This method must be called after read_risk_data, to be able to filter to sources according to the site collection. """ if 'source' in self.oqparam.inputs: logging.info('Reading the composite source model') with self.monitor('reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model( self.oqparam, self.sitecol, self.SourceProcessor, self.monitor, dstore=self.datastore) # we could manage limits here self.job_info = readinput.get_job_info(self.oqparam, self.csm, self.sitecol) self.rlzs_assoc = self.csm.get_rlzs_assoc() logging.info('Total weight of the sources=%s', self.job_info['input_weight']) logging.info('Expected output size=%s', self.job_info['output_weight'])
def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ job_info = hdf5.LiteralAttrs() if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator precalc_id = self.oqparam.hazard_calculation_id if precalc_id is None: # recompute everything precalc = calculators[self.pre_calculator]( self.oqparam, self.monitor("precalculator"), self.datastore.calc_id ) precalc.run() if "scenario" not in self.oqparam.calculation_mode: self.csm = precalc.csm pre_attrs = vars(precalc) for name in ("riskmodel", "assets_by_site"): if name in pre_attrs: setattr(self, name, getattr(precalc, name)) else: # read previously computed data parent = datastore.read(precalc_id) self.datastore.set_parent(parent) # copy missing parameters from the parent params = { name: value for name, value in vars(parent["oqparam"]).items() if name not in vars(self.oqparam) } self.save_params(**params) self.read_risk_data() self.init() else: # we are in a basic calculator self.read_risk_data() if "source" in self.oqparam.inputs: with self.monitor("reading composite source model", autoflush=True): self.csm = readinput.get_composite_source_model(self.oqparam) self.datastore["csm_info"] = self.csm.info self.rup_data = {} # we could manage limits here vars(job_info).update(readinput.get_job_info(self.oqparam, self.csm, self.sitecol)) logging.info("Expected output size=%s", job_info.hazard["output_weight"]) logging.info("Total weight of the sources=%s", job_info.hazard["input_weight"]) self.init() if "source" in self.oqparam.inputs: with self.monitor("managing sources", autoflush=True): self.send_sources() self.manager.store_source_info(self.datastore) attrs = self.datastore.hdf5["composite_source_model"].attrs attrs["weight"] = self.csm.weight attrs["filtered_weight"] = self.csm.filtered_weight attrs["maxweight"] = self.csm.maxweight job_info.hostname = socket.gethostname() if hasattr(self, "riskmodel"): job_info.require_epsilons = bool(self.riskmodel.covs) self.job_info = job_info self.datastore.flush()
def pre_execute(self): """ Read the full source model and sites and build the needed tiles """ self.oqparam = self.job.get_oqparam() source_model_lt = readinput.get_source_model_lt(self.oqparam) source_models = list( readinput.get_source_models(self.oqparam, source_model_lt)) self.parse_risk_model() self.initialize_site_collection() info = readinput.get_job_info(self.oqparam, source_models, self.site_collection) self.imtls = self.oqparam.imtls weight = info['n_sites'] * info['n_levels'] * info['max_realizations'] nblocks = math.ceil(weight / self.oqparam.maximum_tile_weight) self.tiles = list(split_in_blocks(self.site_collection, nblocks)) self.num_tiles = len(self.tiles)
def _print_info(assoc, oqparam, csm, sitecol, filtersources=True, weightsources=True): print(assoc.csm_info) print('See https://github.com/gem/oq-risklib/blob/master/doc/' 'effective-realizations.rst for an explanation') print(assoc) if filtersources or weightsources: info = readinput.get_job_info(oqparam, csm, sitecol) info['n_sources'] = csm.get_num_sources() info['c_matrix'] = humansize( info['n_sites'] * info['n_levels'] * info['n_imts'] * len(assoc) * 8) for k in sorted(info): if k == 'input_weight' and not weightsources: pass else: print(k, info[k])
def pre_execute(self): """ Read the full source model and sites and build the needed tiles """ self.oqparam = self.job.get_oqparam() source_model_lt = readinput.get_source_model_lt(self.oqparam) source_models = list(readinput.get_source_models( self.oqparam, source_model_lt)) self.parse_risk_model() self.initialize_site_collection() info = readinput.get_job_info( self.oqparam, source_models, self.site_collection) self.imtls = self.oqparam.imtls weight = info['n_sites'] * info['n_levels'] * info['max_realizations'] nblocks = math.ceil(weight / self.oqparam.maximum_tile_weight) self.tiles = list(split_in_blocks(self.site_collection, nblocks)) self.num_tiles = len(self.tiles)
def pre_execute(self): """ Initialize risk models, site model and sources """ self.parse_risk_model() self.initialize_site_collection() self.initialize_sources() info = readinput.get_job_info( self.oqparam, self.composite_model, self.site_collection) models.JobInfo.objects.create( oq_job=self.job, num_sites=info['n_sites'], num_realizations=info['max_realizations'], num_imts=info['n_imts'], num_levels=info['n_levels'], input_weight=info['input_weight'], output_weight=info['output_weight']) self.check_limits(info['input_weight'], info['output_weight']) self.init_zeros_ones() return info['input_weight'], info['output_weight']
def _print_info(dstore, filtersources=True, weightsources=True): assoc = dstore["rlzs_assoc"] oqparam = OqParam.from_(dstore.attrs) csm = dstore["composite_source_model"] sitecol = dstore["sitecol"] print(csm.get_info()) print("See https://github.com/gem/oq-risklib/blob/master/doc/" "effective-realizations.rst for an explanation") print(assoc) if filtersources or weightsources: [info] = readinput.get_job_info(oqparam, csm, sitecol) info["n_sources"] = csm.get_num_sources() curve_matrix_size = info["n_sites"] * info["n_levels"] * info["n_imts"] * len(assoc) * 8 for k in info.dtype.fields: if k == "input_weight" and not weightsources: pass else: print(k, info[k]) print("curve_matrix_size", humansize(curve_matrix_size)) if "num_ruptures" in dstore: print(datastore.view("rupture_collections", dstore))
def pre_execute(self): """ Initialize risk models, site model and sources """ self.parse_risk_model() self.initialize_site_collection() self.initialize_sources() info = readinput.get_job_info(self.oqparam, self.composite_model, self.site_collection) models.JobInfo.objects.create( oq_job=self.job, num_sites=info['n_sites'], num_realizations=info['max_realizations'], num_imts=info['n_imts'], num_levels=info['n_levels'], input_weight=info['input_weight'], output_weight=info['output_weight']) self.check_limits(info['input_weight'], info['output_weight']) self.init_zeros_ones() return info['input_weight'], info['output_weight']
def pre_execute(self): """ Read the site collection and the sources. """ if 'exposure' in self.oqparam.inputs: logging.info('Reading the exposure') exposure = readinput.get_exposure(self.oqparam) self.sitecol, _assets = readinput.get_sitecol_assets( self.oqparam, exposure) else: logging.info('Reading the site collection') self.sitecol = readinput.get_site_collection(self.oqparam) logging.info('Reading the effective source models') source_models = list( readinput.get_effective_source_models(self.oqparam, self.sitecol)) self.all_sources = [src for src_model in source_models for trt_model in src_model.trt_models for src in trt_model] self.job_info = readinput.get_job_info( self.oqparam, source_models, self.sitecol)
def _print_info(dstore, filtersources=True, weightsources=True): assoc = dstore['rlzs_assoc'] oqparam = OqParam.from_(dstore.attrs) csm = dstore['composite_source_model'] sitecol = dstore['sitecol'] print(csm.get_info()) print('See https://github.com/gem/oq-risklib/blob/master/doc/' 'effective-realizations.rst for an explanation') print(assoc) if filtersources or weightsources: [info] = readinput.get_job_info(oqparam, csm, sitecol) info['n_sources'] = csm.get_num_sources() curve_matrix_size = (info['n_sites'] * info['n_levels'] * info['n_imts'] * len(assoc) * 8) for k in info.dtype.fields: if k == 'input_weight' and not weightsources: pass else: print(k, info[k]) print('curve_matrix_size', humansize(curve_matrix_size)) if 'num_ruptures' in dstore: print(datastore.view('rupture_collections', dstore))
def pre_execute(self): """ Initialize risk models, site model and sources """ # if you don't use an explicit transaction, errors will be masked # the problem is that Django by default performs implicit transactions # without rollback, see # https://docs.djangoproject.com/en/1.3/topics/db/transactions/ with transaction.commit_on_success(using='job_init'): self.parse_risk_model() with transaction.commit_on_success(using='job_init'): self.initialize_site_collection() with transaction.commit_on_success(using='job_init'): source_models = self.initialize_sources() info = readinput.get_job_info( self.hc, source_models, self.site_collection) with transaction.commit_on_success(using='job_init'): models.JobInfo.objects.create( oq_job=self.job, num_sites=info['n_sites'], num_realizations=info['max_realizations'], num_imts=info['n_imts'], num_levels=info['n_levels'], input_weight=info['input_weight'], output_weight=info['output_weight']) self.check_limits(info['input_weight'], info['output_weight']) self.all_sources = [src for sm in source_models for trt_model in sm.trt_models for src in trt_model] self.imtls = self.hc.intensity_measure_types_and_levels if info['n_levels']: # we can compute hazard curves self.zeros = numpy.array( [numpy.zeros((info['n_sites'], len(self.imtls[imt]))) for imt in sorted(self.imtls)]) self.ones = [numpy.zeros(len(self.imtls[imt]), dtype=float) for imt in sorted(self.imtls)] return info['input_weight'], info['output_weight']
def _print_info(dstore, filtersources=True, weightsources=True): assoc = dstore['rlzs_assoc'] oqparam = dstore['oqparam'] csm = dstore['composite_source_model'] sitecol = dstore['sitecol'] print(csm.get_info()) print('See https://github.com/gem/oq-risklib/blob/master/doc/' 'effective-realizations.rst for an explanation') print(assoc) if filtersources or weightsources: [info] = readinput.get_job_info(oqparam, csm, sitecol) info['n_sources'] = csm.get_num_sources() curve_matrix_size = ( info['n_sites'] * info['n_levels'] * info['n_imts'] * len(assoc) * 8) for k in info.dtype.fields: if k == 'input_weight' and not weightsources: pass else: print(k, info[k]) print('curve_matrix_size', humansize(curve_matrix_size)) if 'num_ruptures' in dstore: print(datastore.view('rupture_collections', dstore))
def pre_execute(self): """ Check if there is a pre_calculator or a previous calculation ID. If yes, read the inputs by invoking the precalculator or by retrieving the previous calculation; if not, read the inputs directly. """ job_info = hdf5.LiteralAttrs() if self.pre_calculator is not None: # the parameter hazard_calculation_id is only meaningful if # there is a precalculator precalc_id = self.oqparam.hazard_calculation_id if precalc_id is None: # recompute everything precalc = calculators[self.pre_calculator]( self.oqparam, self.monitor('precalculator'), self.datastore.calc_id) precalc.run() if 'scenario' not in self.oqparam.calculation_mode: self.csm = precalc.csm pre_attrs = vars(precalc) for name in ('riskmodel', 'assets_by_site'): if name in pre_attrs: setattr(self, name, getattr(precalc, name)) else: # read previously computed data parent = datastore.read(precalc_id) self.datastore.set_parent(parent) # copy missing parameters from the parent params = {name: value for name, value in vars(parent['oqparam']).items() if name not in vars(self.oqparam)} self.save_params(**params) self.read_risk_data() else: # we are in a basic calculator self.read_risk_data() if 'source' in self.oqparam.inputs: with self.monitor( 'reading composite source model', autoflush=True): self.csm = readinput.get_composite_source_model( self.oqparam) self.rlzs_assoc = self.csm.info.get_rlzs_assoc() self.datastore['csm_info'] = self.rlzs_assoc.csm_info self.rup_data = {} # we could manage limits here vars(job_info).update(readinput.get_job_info( self.oqparam, self.csm, self.sitecol)) logging.info('Expected output size=%s', job_info.hazard['output_weight']) logging.info('Total weight of the sources=%s', job_info.hazard['input_weight']) self.init() with self.monitor('managing sources', autoflush=True): self.send_sources() self.manager.store_source_info(self.datastore) attrs = self.datastore.hdf5['composite_source_model'].attrs attrs['weight'] = self.csm.weight attrs['filtered_weight'] = self.csm.filtered_weight attrs['maxweight'] = self.csm.maxweight job_info.hostname = socket.gethostname() if hasattr(self, 'riskmodel'): job_info.require_epsilons = bool(self.riskmodel.covs) self.job_info = job_info self.datastore.flush()