Ejemplo n.º 1
0
    def initialize_sources(self):
        """
        Parse and validation logic trees (source and gsim). Then get all
        sources referenced in the the source model logic tree, create
        :class:`~openquake.db.models.Input` records for all of them, parse
        then, and save the parsed sources to the `parsed_source` table
        (see :class:`openquake.db.models.ParsedSource`).
        """
        logs.log_progress("initializing sources", 2)
        hc = self.job.hazard_calculation

        [smlt] = models.inputs4hcalc(hc.id, input_type='lt_source')
        [gsimlt] = models.inputs4hcalc(hc.id, input_type='lt_gsim')
        source_paths = logictree.read_logic_trees(
            hc.base_path, smlt.path, gsimlt.path)

        src_inputs = []
        for src_path in source_paths:
            full_path = os.path.join(hc.base_path, src_path)

            # Get or reuse the 'source' Input:
            inp = engine2.get_input(
                full_path, 'source', hc.owner, hc.force_inputs)
            src_inputs.append(inp)

            # Associate the source input to the calculation:
            models.Input2hcalc.objects.get_or_create(
                input=inp, hazard_calculation=hc)

            # Associate the source input to the source model logic tree input:
            models.Src2ltsrc.objects.get_or_create(
                hzrd_src=inp, lt_src=smlt, filename=src_path)

        # Now parse the source models and store `pared_source` records:
        for src_inp in src_inputs:
            src_content = StringIO.StringIO(src_inp.model_content.raw_content)
            sm_parser = nrml_parsers.SourceModelParser(src_content)
            src_db_writer = source.SourceDBWriter(
                src_inp, sm_parser.parse(), hc.rupture_mesh_spacing,
                hc.width_of_mfd_bin, hc.area_source_discretization)
            src_db_writer.serialize()
Ejemplo n.º 2
0
def _insert_input_files(params, job, force_inputs):
    """Create uiapi.input records for all input files

    :param dict params: The job config params.
    :param job: The :class:`openquake.db.models.OqJob` instance to use
    :param bool force_inputs: If `True` the model input files will be parsed
        and the resulting content written to the database no matter what.
    """

    inputs_seen = set()

    def ln_input2job(path, input_type):
        """Link identical or newly created input to the given job."""
        digest = _file_digest(path)
        linked_inputs = inputs4job(job.id)
        if any(li.digest == digest and li.input_type == input_type
               for li in linked_inputs):
            return

        in_model = (_identical_input(input_type, digest, job.owner.id)
                    if not force_inputs else None)
        if in_model is None:
            # Save the raw input file contents to the DB:
            model_content = ModelContent()
            with open(path, 'rb') as fh:
                model_content.raw_content = fh.read()
            # Try to guess the content type:
            model_content.content_type = _get_content_type(path)
            model_content.save()

            in_model = Input(path=path, input_type=input_type, owner=job.owner,
                             size=os.path.getsize(path), digest=digest,
                             model_content=model_content)
            in_model.save()

        # Make sure we don't link to the same input more than once.
        if in_model.id not in inputs_seen:
            inputs_seen.add(in_model.id)

            i2j = Input2job(input=in_model, oq_job=job)
            i2j.save()

        return in_model

    # insert input files in input table
    for param_key, file_type in INPUT_FILE_TYPES.items():
        if param_key not in params:
            continue
        path = os.path.join(params['BASE_PATH'], params[param_key])
        input_obj = ln_input2job(path, file_type)

        if file_type == "lt_source":
            # collect relative paths to source model files
            # that are referenced by the logic tree
            source_models = logictree.read_logic_trees(
                params['BASE_PATH'], params['SOURCE_MODEL_LOGIC_TREE_FILE'],
                params['GMPE_LOGIC_TREE_FILE']
            )
            # insert source model files
            for srcrelpath in source_models:
                abspath = os.path.join(params['BASE_PATH'], srcrelpath)
                hzrd_src_input = ln_input2job(abspath, "source")
                Src2ltsrc.objects.get_or_create(
                    hzrd_src=hzrd_src_input, lt_src=input_obj,
                    defaults={'filename': srcrelpath}
                )