Example #1
0
    def initialize_sources(self):
        """
        Parse and validation logic trees (source and gsim). Then get all
        sources referenced in the the source model logic tree, create
        :class:`~openquake.engine.db.models.Input` records for all of them,
        parse then, and save the parsed sources to the `parsed_source` table
        (see :class:`openquake.engine.db.models.ParsedSource`).
        """
        logs.LOG.progress("initializing sources")

        [smlt] = models.inputs4hcalc(
            self.hc.id, input_type='source_model_logic_tree')
        [gsimlt] = models.inputs4hcalc(
            self.hc.id, input_type='gsim_logic_tree')
        source_paths = logictree.read_logic_trees(
            self.hc.base_path, smlt.path, gsimlt.path)

        src_inputs = []
        for src_path in source_paths:
            full_path = os.path.join(self.hc.base_path, src_path)

            # Get or reuse the 'source' Input:
            inp = engine2.get_input(
                full_path, 'source', self.hc.owner, self.hc.force_inputs)
            src_inputs.append(inp)

            # Associate the source input to the calculation:
            models.Input2hcalc.objects.get_or_create(
                input=inp, hazard_calculation=self.hc)

            # Associate the source input to the source model logic tree input:
            try:
                models.Src2ltsrc.objects.get(lt_src=smlt, filename=src_path)
            except ObjectDoesNotExist:
                # If it doesn't exist, this is a new input and we're not
                # reusing an old one which is identical.
                # Only in this case do we parse the sources and populate
                # `hzrdi.parsed_source`.
                models.Src2ltsrc.objects.create(hzrd_src=inp, lt_src=smlt,
                                                filename=src_path)
                src_content = StringIO.StringIO(
                    inp.model_content.raw_content_ascii)
                sm_parser = nrml_parsers.SourceModelParser(src_content)
                src_db_writer = source.SourceDBWriter(
                    inp, sm_parser.parse(), self.hc.rupture_mesh_spacing,
                    self.hc.width_of_mfd_bin,
                    self.hc.area_source_discretization
                )
                src_db_writer.serialize()
Example #2
0
    def initialize_sources(self):
        """
        Parse and validation source logic trees. Save the parsed
        sources to the `parsed_source` table (see
        :class:`openquake.engine.db.models.ParsedSource`).
        """
        logs.LOG.progress("initializing sources")

        for src_path in logictree.read_logic_trees(self.hc):
            source.SourceDBWriter(
                self.job, src_path,
                nrml_parsers.SourceModelParser(
                    os.path.join(self.hc.base_path, src_path)).parse(),
                self.hc.rupture_mesh_spacing, self.hc.width_of_mfd_bin,
                self.hc.area_source_discretization,
                self.get_source_filter_condition()).serialize()
Example #3
0
    def initialize_sources(self):
        """
        Parse and validation source logic trees. Save the parsed
        sources to the `parsed_source` table (see
        :class:`openquake.engine.db.models.ParsedSource`).
        """
        logs.LOG.progress("initializing sources")

        for src_path in logictree.read_logic_trees(self.hc):
            source.SourceDBWriter(
                self.job,
                src_path,
                nrml_parsers.SourceModelParser(
                    os.path.join(self.hc.base_path, src_path)).parse(),
                self.hc.rupture_mesh_spacing,
                self.hc.width_of_mfd_bin,
                self.hc.area_source_discretization,
                self.get_source_filter_condition()).serialize()
Example #4
0
 def initialize_sources(self):
     """
     Parse and validate source logic trees
     """
     logs.LOG.progress("initializing sources")
     for src_path in logictree.read_logic_trees(self.hc):
         for src_nrml in nrml_parsers.SourceModelParser(
                 os.path.join(self.hc.base_path, src_path)).parse():
             src = source.nrml_to_hazardlib(
                 src_nrml,
                 self.hc.rupture_mesh_spacing,
                 self.hc.width_of_mfd_bin,
                 self.hc.area_source_discretization)
             if self.filtered_sites(src):
                 if isinstance(src_nrml, PointSource):
                     self.sources_per_model[src_path, 'point'].append(src)
                 else:
                     self.sources_per_model[src_path, 'other'].append(src)
Example #5
0
    def initialize_sources(self):
        """
        Parse and validation logic trees (source and gsim). Then get all
        sources referenced in the the source model logic tree, create
        :class:`~openquake.engine.db.models.Input` records for all of them,
        parse then, and save the parsed sources to the `parsed_source` table
        (see :class:`openquake.engine.db.models.ParsedSource`).
        """
        logs.LOG.progress("initializing sources")

        [smlt] = models.inputs4hcalc(
            self.hc.id, input_type='source_model_logic_tree')
        [gsimlt] = models.inputs4hcalc(
            self.hc.id, input_type='gsim_logic_tree')
        source_paths = logictree.read_logic_trees(
            self.hc.base_path, smlt.path, gsimlt.path)

        for src_path in source_paths:
            full_path = os.path.join(self.hc.base_path, src_path)

            # Get the 'source' Input:
            inp = engine.get_input(full_path, 'source', self.hc.owner)

            # Associate the source input to the calculation:
            models.Input2hcalc.objects.get_or_create(
                input=inp, hazard_calculation=self.hc)

            models.Src2ltsrc.objects.create(hzrd_src=inp, lt_src=smlt,
                                            filename=src_path)
            src_content = StringIO.StringIO(
                inp.model_content.raw_content_ascii)
            sm_parser = nrml_parsers.SourceModelParser(src_content)

            # Covert
            src_db_writer = source.SourceDBWriter(
                inp, sm_parser.parse(), self.hc.rupture_mesh_spacing,
                self.hc.width_of_mfd_bin,
                self.hc.area_source_discretization
            )
            src_db_writer.serialize()
Example #6
0
def _insert_input_files(params, job, force_inputs):
    """Create uiapi.input records for all input files

    :param dict params: The job config params.
    :param job: The :class:`openquake.engine.db.models.OqJob` instance to use
    :param bool force_inputs: If `True` the model input files will be parsed
        and the resulting content written to the database no matter what.
    """

    inputs_seen = set()

    def ln_input2job(path, input_type):
        """Link identical or newly created input to the given job."""
        digest = _file_digest(path)
        linked_inputs = inputs4job(job.id)
        if any(li.digest == digest and li.input_type == input_type
               for li in linked_inputs):
            return

        in_model = (_identical_input(input_type, digest, job.owner.id)
                    if not force_inputs else None)
        if in_model is None:
            # Save the raw input file contents to the DB:
            model_content = ModelContent()
            with open(path, 'rb') as fh:
                model_content.raw_content = fh.read()
            # Try to guess the content type:
            model_content.content_type = _get_content_type(path)
            model_content.save()

            in_model = Input(path=path, input_type=input_type, owner=job.owner,
                             size=os.path.getsize(path), digest=digest,
                             model_content=model_content)
            in_model.save()

        # Make sure we don't link to the same input more than once.
        if in_model.id not in inputs_seen:
            inputs_seen.add(in_model.id)

            i2j = Input2job(input=in_model, oq_job=job)
            i2j.save()

        return in_model

    # insert input files in input table
    for param_key, file_type in INPUT_FILE_TYPES.items():
        if param_key not in params:
            continue
        path = os.path.join(params['BASE_PATH'], params[param_key])
        input_obj = ln_input2job(path, file_type)

        if file_type == "lt_source":
            # collect relative paths to source model files
            # that are referenced by the logic tree
            source_models = logictree.read_logic_trees(
                params['BASE_PATH'], params['SOURCE_MODEL_LOGIC_TREE_FILE'],
                params['GMPE_LOGIC_TREE_FILE']
            )
            # insert source model files
            for srcrelpath in source_models:
                abspath = os.path.join(params['BASE_PATH'], srcrelpath)
                hzrd_src_input = ln_input2job(abspath, "source")
                Src2ltsrc.objects.get_or_create(
                    hzrd_src=hzrd_src_input, lt_src=input_obj,
                    defaults={'filename': srcrelpath}
                )