예제 #1
0
파일: extraction_api.py 프로젝트: autt/ert
def dump_to_new_storage(reference=None, rdb_session=None, blob_session=None):

    start_time = time.time()
    logger.debug("Starting extraction...")

    if rdb_session is None:
        rdb_session = ERT_STORAGE.RdbSession()

    if blob_session is None:
        blob_session = ERT_STORAGE.BlobSession()

    rdb_api = RdbApi(session=rdb_session)
    blob_api = BlobApi(session=blob_session)

    try:
        priors = _extract_and_dump_priors(
            rdb_api=rdb_api) if reference is None else []

        ensemble = _create_ensemble(rdb_api,
                                    reference=reference,
                                    priors=priors)
        _extract_and_dump_observations(rdb_api=rdb_api, blob_api=blob_api)

        _extract_and_dump_parameters(
            rdb_api=rdb_api,
            blob_api=blob_api,
            ensemble_name=ensemble.name,
            priors=priors,
        )
        _extract_and_dump_responses(rdb_api=rdb_api,
                                    blob_api=blob_api,
                                    ensemble_name=ensemble.name)
        _extract_and_dump_update_data(ensemble.id, ensemble.name, rdb_api,
                                      blob_api)

        rdb_session.commit()
        blob_session.commit()
        ensemble_name = ensemble.name

        end_time = time.time()
        logger.debug(
            "Extraction done... (Took {:.2f} seconds)".format(end_time -
                                                              start_time))
        logger.debug("All ensembles in database: {}".format(", ".join(
            [ensemble.name for ensemble in rdb_api.get_all_ensembles()])))

    except:
        rdb_session.rollback()
        blob_session.rollback()
        raise
    finally:
        rdb_session.close()
        blob_session.close()

    return ensemble_name
예제 #2
0
def apis(initialize_databases):
    rdb_session = ERT_STORAGE.RdbSession()
    blob_session = ERT_STORAGE.BlobSession()

    rdb_api = RdbApi(session=rdb_session)
    blob_api = BlobApi(session=blob_session)

    try:
        yield rdb_api, blob_api
    finally:
        rdb_session.rollback()
        blob_session.rollback()
        rdb_session.close()
        blob_session.close()
예제 #3
0
def db_apis(populated_database):
    db_lookup = populated_database

    rdb_session = ERT_STORAGE.RdbSession()
    blob_session = ERT_STORAGE.BlobSession()

    rdb_api = RdbApi(session=rdb_session)
    blob_api = BlobApi(session=blob_session)

    try:
        yield rdb_api, blob_api, db_lookup
    finally:
        rdb_session.rollback()
        blob_session.rollback()
        rdb_session.close()
        blob_session.close()
예제 #4
0
 def session(self):
     """Provide a transactional scope around a series of operations."""
     rdb_session = ERT_STORAGE.RdbSession()
     blob_session = ERT_STORAGE.BlobSession()
     rdb_api = RdbApi(session=rdb_session)
     blob_api = BlobApi(session=blob_session)
     try:
         yield StorageApi(rdb_api, blob_api)
         rdb_session.commit()
         blob_session.commit()
     except:
         rdb_session.rollback()
         blob_session.rollback()
         raise
     finally:
         rdb_session.close()
         blob_session.close()
예제 #5
0
def populated_database(initialize_databases):
    rdb_session = ERT_STORAGE.RdbSession()
    blob_session = ERT_STORAGE.BlobSession()

    rdb_api = RdbApi(session=rdb_session)
    blob_api = BlobApi(session=blob_session)

    def add_blob(data):
        ret = blob_api.add_blob(data)
        return ret.id

    db_lookup = {}
    ######## add priors ########
    prior_key1 = rdb_api.add_prior(
        "group", "key1", "function", ["paramA", "paramB"], [0.1, 0.2]
    )
    prior_key2 = rdb_api.add_prior(
        "group", "key2", "function", ["paramA", "paramB"], [0.3, 0.4]
    )
    prior_key3 = rdb_api.add_prior(
        "group", "key3", "function", ["paramA", "paramB"], [0.5, 0.6]
    )
    prior_A = rdb_api.add_prior("G", "A", "function", ["paramA", "paramB"], [0.5, 0.6])

    ######## add ensemble ########
    ensemble = rdb_api.add_ensemble(
        name="ensemble_name", priors=[prior_key1, prior_key2, prior_key3]
    )
    db_lookup["ensemble"] = ensemble.id
    db_lookup["ensemble_timestamp"] = ensemble.time_created
    ######## add parameteredefinitionss ########
    parameter_def_A_G = rdb_api.add_parameter_definition(
        "A", "G", "ensemble_name", prior=prior_A
    )
    parameter_def_B_G = rdb_api.add_parameter_definition("B", "G", "ensemble_name")
    parameter_def_key1_group = rdb_api.add_parameter_definition(
        "key1", "group", "ensemble_name", prior=prior_key1
    )
    db_lookup["parameter_def_A_G"] = parameter_def_A_G.id
    db_lookup["parameter_def_key1_group"] = parameter_def_key1_group.id

    ######## add observations ########
    observation_one = rdb_api.add_observation(
        name="observation_one",
        key_indexes_ref=add_blob([0, 3]),
        data_indexes_ref=add_blob([2, 3]),
        values_ref=add_blob([10.1, 10.2]),
        stds_ref=add_blob([1, 3]),
    )
    observation_one.add_attribute("region", "1")

    observation_two_first = rdb_api.add_observation(
        name="observation_two_first",
        key_indexes_ref=add_blob(["2000-01-01 20:01:01"]),
        data_indexes_ref=add_blob([4]),
        values_ref=add_blob([10.3]),
        stds_ref=add_blob([2]),
    )

    observation_two_second = rdb_api.add_observation(
        name="observation_two_second",
        key_indexes_ref=add_blob(["2000-01-02 20:01:01"]),
        data_indexes_ref=add_blob([5]),
        values_ref=add_blob([10.4]),
        stds_ref=add_blob([2.5]),
    )

    ######## add response definitions ########
    response_definition_one = rdb_api.add_response_definition(
        name="response_one",
        indexes_ref=add_blob([3, 5, 8, 9]),
        ensemble_name=ensemble.name,
    )

    response_definition_two = rdb_api.add_response_definition(
        name="response_two",
        indexes_ref=add_blob(
            [
                "2000-01-01 20:01:01",
                "2000-01-02 20:01:01",
                "2000-01-02 20:01:01",
                "2000-01-02 20:01:01",
                "2000-01-02 20:01:01",
                "2000-01-02 20:01:01",
            ]
        ),
        ensemble_name=ensemble.name,
    )
    db_lookup["response_defition_one"] = response_definition_one.id

    ######## observation response definition links ########
    obs_res_def_link = rdb_api._add_observation_response_definition_link(
        observation_id=observation_one.id,
        response_definition_id=response_definition_one.id,
        active_ref=add_blob([True, False]),
        update_id=None,
    )

    rdb_api._add_observation_response_definition_link(
        observation_id=observation_two_first.id,
        response_definition_id=response_definition_two.id,
        active_ref=add_blob([True]),
        update_id=None,
    )

    rdb_api._add_observation_response_definition_link(
        observation_id=observation_two_second.id,
        response_definition_id=response_definition_two.id,
        active_ref=add_blob([True]),
        update_id=None,
    )

    ######## add realizations ########
    realization_0 = rdb_api.add_realization(0, ensemble.name)
    realization_1 = rdb_api.add_realization(1, ensemble.name)
    db_lookup["realization_0"] = realization_0.id

    def add_data(realization, response_def, ens):
        response_one = rdb_api.add_response(
            name="response_one",
            values_ref=add_blob([11.1, 11.2, 9.9, 9.3]),
            realization_index=realization.index,
            ensemble_name=ens.name,
        )
        rdb_api._add_misfit(200, obs_res_def_link.id, response_one.id)

        rdb_api.add_response(
            name="response_two",
            values_ref=add_blob([12.1, 12.2, 11.1, 11.2, 9.9, 9.3]),
            realization_index=realization.index,
            ensemble_name=ens.name,
        )

        rdb_api.add_parameter("A", "G", add_blob(1), realization.index, "ensemble_name")
        rdb_api.add_parameter("B", "G", add_blob(2), realization.index, "ensemble_name")
        rdb_api.add_parameter(
            "key1", "group", add_blob(2), realization.index, "ensemble_name"
        )

    add_data(realization_0, response_def=response_definition_one, ens=ensemble)
    add_data(realization_1, response_def=response_definition_one, ens=ensemble)

    ######## add blob #########
    data_blob = add_blob([0, 1, 2, 3])
    db_lookup["data_blob"] = data_blob

    rdb_session.commit()
    blob_session.commit()

    rdb_session.close()
    blob_session.close()

    yield db_lookup