def initialize_sources(self): """ Parse and validation logic trees (source and gsim). Then get all sources referenced in the the source model logic tree, create :class:`~openquake.engine.db.models.Input` records for all of them, parse then, and save the parsed sources to the `parsed_source` table (see :class:`openquake.engine.db.models.ParsedSource`). """ logs.LOG.progress("initializing sources") [smlt] = models.inputs4hcalc( self.hc.id, input_type='source_model_logic_tree') [gsimlt] = models.inputs4hcalc( self.hc.id, input_type='gsim_logic_tree') source_paths = logictree.read_logic_trees( self.hc.base_path, smlt.path, gsimlt.path) src_inputs = [] for src_path in source_paths: full_path = os.path.join(self.hc.base_path, src_path) # Get or reuse the 'source' Input: inp = engine2.get_input( full_path, 'source', self.hc.owner, self.hc.force_inputs) src_inputs.append(inp) # Associate the source input to the calculation: models.Input2hcalc.objects.get_or_create( input=inp, hazard_calculation=self.hc) # Associate the source input to the source model logic tree input: try: models.Src2ltsrc.objects.get(lt_src=smlt, filename=src_path) except ObjectDoesNotExist: # If it doesn't exist, this is a new input and we're not # reusing an old one which is identical. # Only in this case do we parse the sources and populate # `hzrdi.parsed_source`. models.Src2ltsrc.objects.create(hzrd_src=inp, lt_src=smlt, filename=src_path) src_content = StringIO.StringIO( inp.model_content.raw_content_ascii) sm_parser = nrml_parsers.SourceModelParser(src_content) src_db_writer = source.SourceDBWriter( inp, sm_parser.parse(), self.hc.rupture_mesh_spacing, self.hc.width_of_mfd_bin, self.hc.area_source_discretization ) src_db_writer.serialize()
def initialize_sources(self): """ Parse and validation source logic trees. Save the parsed sources to the `parsed_source` table (see :class:`openquake.engine.db.models.ParsedSource`). """ logs.LOG.progress("initializing sources") for src_path in logictree.read_logic_trees(self.hc): source.SourceDBWriter( self.job, src_path, nrml_parsers.SourceModelParser( os.path.join(self.hc.base_path, src_path)).parse(), self.hc.rupture_mesh_spacing, self.hc.width_of_mfd_bin, self.hc.area_source_discretization, self.get_source_filter_condition()).serialize()
def initialize_sources(self): """ Parse and validate source logic trees """ logs.LOG.progress("initializing sources") for src_path in logictree.read_logic_trees(self.hc): for src_nrml in nrml_parsers.SourceModelParser( os.path.join(self.hc.base_path, src_path)).parse(): src = source.nrml_to_hazardlib( src_nrml, self.hc.rupture_mesh_spacing, self.hc.width_of_mfd_bin, self.hc.area_source_discretization) if self.filtered_sites(src): if isinstance(src_nrml, PointSource): self.sources_per_model[src_path, 'point'].append(src) else: self.sources_per_model[src_path, 'other'].append(src)
def initialize_sources(self): """ Parse and validation logic trees (source and gsim). Then get all sources referenced in the the source model logic tree, create :class:`~openquake.engine.db.models.Input` records for all of them, parse then, and save the parsed sources to the `parsed_source` table (see :class:`openquake.engine.db.models.ParsedSource`). """ logs.LOG.progress("initializing sources") [smlt] = models.inputs4hcalc( self.hc.id, input_type='source_model_logic_tree') [gsimlt] = models.inputs4hcalc( self.hc.id, input_type='gsim_logic_tree') source_paths = logictree.read_logic_trees( self.hc.base_path, smlt.path, gsimlt.path) for src_path in source_paths: full_path = os.path.join(self.hc.base_path, src_path) # Get the 'source' Input: inp = engine.get_input(full_path, 'source', self.hc.owner) # Associate the source input to the calculation: models.Input2hcalc.objects.get_or_create( input=inp, hazard_calculation=self.hc) models.Src2ltsrc.objects.create(hzrd_src=inp, lt_src=smlt, filename=src_path) src_content = StringIO.StringIO( inp.model_content.raw_content_ascii) sm_parser = nrml_parsers.SourceModelParser(src_content) # Covert src_db_writer = source.SourceDBWriter( inp, sm_parser.parse(), self.hc.rupture_mesh_spacing, self.hc.width_of_mfd_bin, self.hc.area_source_discretization ) src_db_writer.serialize()
def _insert_input_files(params, job, force_inputs): """Create uiapi.input records for all input files :param dict params: The job config params. :param job: The :class:`openquake.engine.db.models.OqJob` instance to use :param bool force_inputs: If `True` the model input files will be parsed and the resulting content written to the database no matter what. """ inputs_seen = set() def ln_input2job(path, input_type): """Link identical or newly created input to the given job.""" digest = _file_digest(path) linked_inputs = inputs4job(job.id) if any(li.digest == digest and li.input_type == input_type for li in linked_inputs): return in_model = (_identical_input(input_type, digest, job.owner.id) if not force_inputs else None) if in_model is None: # Save the raw input file contents to the DB: model_content = ModelContent() with open(path, 'rb') as fh: model_content.raw_content = fh.read() # Try to guess the content type: model_content.content_type = _get_content_type(path) model_content.save() in_model = Input(path=path, input_type=input_type, owner=job.owner, size=os.path.getsize(path), digest=digest, model_content=model_content) in_model.save() # Make sure we don't link to the same input more than once. if in_model.id not in inputs_seen: inputs_seen.add(in_model.id) i2j = Input2job(input=in_model, oq_job=job) i2j.save() return in_model # insert input files in input table for param_key, file_type in INPUT_FILE_TYPES.items(): if param_key not in params: continue path = os.path.join(params['BASE_PATH'], params[param_key]) input_obj = ln_input2job(path, file_type) if file_type == "lt_source": # collect relative paths to source model files # that are referenced by the logic tree source_models = logictree.read_logic_trees( params['BASE_PATH'], params['SOURCE_MODEL_LOGIC_TREE_FILE'], params['GMPE_LOGIC_TREE_FILE'] ) # insert source model files for srcrelpath in source_models: abspath = os.path.join(params['BASE_PATH'], srcrelpath) hzrd_src_input = ln_input2job(abspath, "source") Src2ltsrc.objects.get_or_create( hzrd_src=hzrd_src_input, lt_src=input_obj, defaults={'filename': srcrelpath} )