def initialize_databases(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp("database")
rdb_url = f"sqlite:///{tmp_path}/entities.db"
blob_url = f"sqlite:///{tmp_path}/blobs.db"
ERT_STORAGE.initialize(rdb_url=rdb_url, blob_url=blob_url)
def dump_to_new_storage(reference=None, rdb_session=None, blob_session=None):
start_time = time.time()
logger.debug("Starting extraction...")
if rdb_session is None:
rdb_session = ERT_STORAGE.RdbSession()
if blob_session is None:
blob_session = ERT_STORAGE.BlobSession()
rdb_api = RdbApi(session=rdb_session)
blob_api = BlobApi(session=blob_session)
try:
priors = _extract_and_dump_priors(
rdb_api=rdb_api) if reference is None else []
ensemble = _create_ensemble(rdb_api,
reference=reference,
priors=priors)
_extract_and_dump_observations(rdb_api=rdb_api, blob_api=blob_api)
_extract_and_dump_parameters(
rdb_api=rdb_api,
blob_api=blob_api,
ensemble_name=ensemble.name,
priors=priors,
)
_extract_and_dump_responses(rdb_api=rdb_api,
blob_api=blob_api,
ensemble_name=ensemble.name)
_extract_and_dump_update_data(ensemble.id, ensemble.name, rdb_api,
blob_api)
rdb_session.commit()
blob_session.commit()
ensemble_name = ensemble.name
end_time = time.time()
logger.debug(
"Extraction done... (Took {:.2f} seconds)".format(end_time -
start_time))
logger.debug("All ensembles in database: {}".format(", ".join(
[ensemble.name for ensemble in rdb_api.get_all_ensembles()])))
except:
rdb_session.rollback()
blob_session.rollback()
raise
finally:
rdb_session.close()
blob_session.close()
return ensemble_name
def apis(initialize_databases):
rdb_session = ERT_STORAGE.RdbSession()
blob_session = ERT_STORAGE.BlobSession()
rdb_api = RdbApi(session=rdb_session)
blob_api = BlobApi(session=blob_session)
try:
yield rdb_api, blob_api
finally:
rdb_session.rollback()
blob_session.rollback()
rdb_session.close()
blob_session.close()
def db_apis(populated_database):
db_lookup = populated_database
rdb_session = ERT_STORAGE.RdbSession()
blob_session = ERT_STORAGE.BlobSession()
rdb_api = RdbApi(session=rdb_session)
blob_api = BlobApi(session=blob_session)
try:
yield rdb_api, blob_api, db_lookup
finally:
rdb_session.rollback()
blob_session.rollback()
rdb_session.close()
blob_session.close()
def session(self):
"""Provide a transactional scope around a series of operations."""
rdb_session = ERT_STORAGE.RdbSession()
blob_session = ERT_STORAGE.BlobSession()
rdb_api = RdbApi(session=rdb_session)
blob_api = BlobApi(session=blob_session)
try:
yield StorageApi(rdb_api, blob_api)
rdb_session.commit()
blob_session.commit()
except:
rdb_session.rollback()
blob_session.rollback()
raise
finally:
rdb_session.close()
blob_session.close()
def api(initialize_databases):
session = ERT_STORAGE.Session()
api = RdbApi(session=session)
try:
yield api
finally:
session.rollback()
session.close()
def db_api(populated_database):
db_lookup = populated_database
session = ERT_STORAGE.Session()
api = RdbApi(session=session)
try:
yield api, db_lookup
finally:
session.rollback()
session.close()
def session(self):
"""Provide a transactional scope around a series of operations."""
session = ERT_STORAGE.Session()
try:
yield StorageApi(session)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
def initialize_databases(tmp_path_factory):
tmp_path = tmp_path_factory.mktemp("database")
url = f"sqlite:///{tmp_path}/ert_storage.db"
ERT_STORAGE.initialize(url=url)
def storage_api(db_api):
api, db_lookup = db_api
yield StorageApi(session=ERT_STORAGE.Session()), db_lookup
def populated_database(initialize_databases):
session = ERT_STORAGE.Session()
api = RdbApi(session=session)
db_lookup = {}
prior_key1 = api.add_prior("group", "key1", "function",
["paramA", "paramB"], [0.1, 0.2])
prior_key2 = api.add_prior("group", "key2", "function",
["paramA", "paramB"], [0.3, 0.4])
prior_key3 = api.add_prior("group", "key3", "function",
["paramA", "paramB"], [0.5, 0.6])
prior_a = api.add_prior("G", "A", "function", ["paramA", "paramB"],
[0.5, 0.6])
######## add ensemble ########
ensemble = api.add_ensemble(name="ensemble_name",
priors=[prior_key1, prior_key2, prior_key3])
db_lookup["ensemble"] = ensemble.id
db_lookup["ensemble_timestamp"] = ensemble.time_created
######## add parameteredefinitionss ########
parameter_def_A_G = api.add_parameter_definition("A",
"G",
"ensemble_name",
prior=prior_a)
parameter_def_B_G = api.add_parameter_definition("B", "G", "ensemble_name")
parameter_def_key1_group = api.add_parameter_definition("key1",
"group",
"ensemble_name",
prior=prior_key1)
db_lookup["parameter_def_A_G"] = parameter_def_A_G.id
db_lookup["parameter_def_key1_group"] = parameter_def_key1_group.id
######## add observations ########
observation_one = api.add_observation(
name="observation_one",
key_indices=[0, 3],
data_indices=[2, 3],
values=[10.1, 10.2],
errors=[1, 3],
)
observation_one.add_attribute("region", "1")
observation_two_first = api.add_observation(
name="observation_two_first",
key_indices=["2000-01-01 20:01:01"],
data_indices=[4],
values=[10.3],
errors=[2],
)
observation_two_second = api.add_observation(
name="observation_two_second",
key_indices=["2000-01-02 20:01:01"],
data_indices=[5],
values=[10.4],
errors=[2.5],
)
######## add response definitions ########
response_definition_one = api.add_response_definition(
name="response_one",
indices=[3, 5, 8, 9],
ensemble_name=ensemble.name,
)
response_definition_two = api.add_response_definition(
name="response_two",
indices=[
"2000-01-01 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
],
ensemble_name=ensemble.name,
)
db_lookup["response_definition_one"] = response_definition_one.id
######## observation response definition links ########
obs_res_def_link = api._add_observation_response_definition_link(
observation_id=observation_one.id,
response_definition_id=response_definition_one.id,
active=[True, False],
update_id=None,
)
api._add_observation_response_definition_link(
observation_id=observation_two_first.id,
response_definition_id=response_definition_two.id,
active=[True],
update_id=None,
)
api._add_observation_response_definition_link(
observation_id=observation_two_second.id,
response_definition_id=response_definition_two.id,
active=[True],
update_id=None,
)
######## add realizations ########
realization_0 = api.add_realization(0, ensemble.name)
realization_1 = api.add_realization(1, ensemble.name)
db_lookup["realization_0"] = realization_0.id
def add_data(realization, ens):
response_one = api.add_response(
name="response_one",
values=[11.1, 11.2, 9.9, 9.3],
realization_index=realization.index,
ensemble_name=ens.name,
)
api._add_misfit(200, obs_res_def_link.id, response_one.id)
api.add_response(
name="response_two",
values=[12.1, 12.2, 11.1, 11.2, 9.9, 9.3],
realization_index=realization.index,
ensemble_name=ens.name,
)
api.add_parameter("A", "G", 1, realization.index, "ensemble_name")
api.add_parameter("B", "G", 2, realization.index, "ensemble_name")
api.add_parameter("key1", "group", 2, realization.index,
"ensemble_name")
add_data(realization_0, ens=ensemble)
add_data(realization_1, ens=ensemble)
session.commit()
session.close()
yield db_lookup
def initialize_databases():
from ert_shared.storage import ERT_STORAGE
ERT_STORAGE.initialize()
def __init__(self, rdb_url, blob_url, secure=True):
ERT_STORAGE.initialize(rdb_url=rdb_url, blob_url=blob_url)
app = flask.Flask("ert http api")
self.app = app
if secure:
self.authtoken = generate_authtoken()
@app.before_request
def check_auth():
if request.authorization is None:
abort(401)
un = request.authorization["username"]
pw = request.authorization["password"]
if un != "__token__" or pw != self.authtoken:
abort(401)
self.app.add_url_rule("/ensembles", "ensembles", self.ensembles)
self.app.add_url_rule(
"/ensembles/<ensemble_id>", "ensemble", self.ensemble_by_id
)
self.app.add_url_rule(
"/ensembles/<ensemble_id>/realizations/<realization_idx>",
"realization",
self.realization_by_id,
)
self.app.add_url_rule(
"/ensembles/<ensemble_id>/responses/<response_name>",
"response",
self.response_by_name,
)
self.app.add_url_rule(
"/ensembles/<ensemble_id>/responses/<response_name>/data",
"response_data",
self.response_data_by_name,
)
self.app.add_url_rule(
"/ensembles/<ensemble_id>/parameters/<parameter_def_id>",
"parameter",
self.parameter_by_id,
)
self.app.add_url_rule(
"/ensembles/<ensemble_id>/parameters/<parameter_def_id>/data",
"parameter_data",
self.parameter_data_by_id,
)
self.app.add_url_rule("/data/<int:data_id>", "data", self.data)
self.app.add_url_rule(
"/observation/<name>",
"get_observation",
self.get_observation,
methods=["GET"],
)
self.app.add_url_rule(
"/observation/<name>/attributes",
"get_observation_attributes",
self.get_observation_attributes,
methods=["GET"],
)
self.app.add_url_rule(
"/observation/<name>/attributes",
"set_observation_attributes",
self.set_observation_attributes,
methods=["POST"],
)
self.app.add_url_rule("/shutdown", "shutdown", self.shutdown, methods=["POST"])
self.app.add_url_rule(
"/schema.json",
"schema",
self.schema,
methods=["GET"],
)
@app.route("/healthcheck")
def healthcheck():
return jsonify({"date": datetime.datetime.now().isoformat()})
def populated_database(initialize_databases):
rdb_session = ERT_STORAGE.RdbSession()
blob_session = ERT_STORAGE.BlobSession()
rdb_api = RdbApi(session=rdb_session)
blob_api = BlobApi(session=blob_session)
def add_blob(data):
ret = blob_api.add_blob(data)
return ret.id
db_lookup = {}
######## add priors ########
prior_key1 = rdb_api.add_prior(
"group", "key1", "function", ["paramA", "paramB"], [0.1, 0.2]
)
prior_key2 = rdb_api.add_prior(
"group", "key2", "function", ["paramA", "paramB"], [0.3, 0.4]
)
prior_key3 = rdb_api.add_prior(
"group", "key3", "function", ["paramA", "paramB"], [0.5, 0.6]
)
prior_A = rdb_api.add_prior("G", "A", "function", ["paramA", "paramB"], [0.5, 0.6])
######## add ensemble ########
ensemble = rdb_api.add_ensemble(
name="ensemble_name", priors=[prior_key1, prior_key2, prior_key3]
)
db_lookup["ensemble"] = ensemble.id
db_lookup["ensemble_timestamp"] = ensemble.time_created
######## add parameteredefinitionss ########
parameter_def_A_G = rdb_api.add_parameter_definition(
"A", "G", "ensemble_name", prior=prior_A
)
parameter_def_B_G = rdb_api.add_parameter_definition("B", "G", "ensemble_name")
parameter_def_key1_group = rdb_api.add_parameter_definition(
"key1", "group", "ensemble_name", prior=prior_key1
)
db_lookup["parameter_def_A_G"] = parameter_def_A_G.id
db_lookup["parameter_def_key1_group"] = parameter_def_key1_group.id
######## add observations ########
observation_one = rdb_api.add_observation(
name="observation_one",
key_indexes_ref=add_blob([0, 3]),
data_indexes_ref=add_blob([2, 3]),
values_ref=add_blob([10.1, 10.2]),
stds_ref=add_blob([1, 3]),
)
observation_one.add_attribute("region", "1")
observation_two_first = rdb_api.add_observation(
name="observation_two_first",
key_indexes_ref=add_blob(["2000-01-01 20:01:01"]),
data_indexes_ref=add_blob([4]),
values_ref=add_blob([10.3]),
stds_ref=add_blob([2]),
)
observation_two_second = rdb_api.add_observation(
name="observation_two_second",
key_indexes_ref=add_blob(["2000-01-02 20:01:01"]),
data_indexes_ref=add_blob([5]),
values_ref=add_blob([10.4]),
stds_ref=add_blob([2.5]),
)
######## add response definitions ########
response_definition_one = rdb_api.add_response_definition(
name="response_one",
indexes_ref=add_blob([3, 5, 8, 9]),
ensemble_name=ensemble.name,
)
response_definition_two = rdb_api.add_response_definition(
name="response_two",
indexes_ref=add_blob(
[
"2000-01-01 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
"2000-01-02 20:01:01",
]
),
ensemble_name=ensemble.name,
)
db_lookup["response_defition_one"] = response_definition_one.id
######## observation response definition links ########
obs_res_def_link = rdb_api._add_observation_response_definition_link(
observation_id=observation_one.id,
response_definition_id=response_definition_one.id,
active_ref=add_blob([True, False]),
update_id=None,
)
rdb_api._add_observation_response_definition_link(
observation_id=observation_two_first.id,
response_definition_id=response_definition_two.id,
active_ref=add_blob([True]),
update_id=None,
)
rdb_api._add_observation_response_definition_link(
observation_id=observation_two_second.id,
response_definition_id=response_definition_two.id,
active_ref=add_blob([True]),
update_id=None,
)
######## add realizations ########
realization_0 = rdb_api.add_realization(0, ensemble.name)
realization_1 = rdb_api.add_realization(1, ensemble.name)
db_lookup["realization_0"] = realization_0.id
def add_data(realization, response_def, ens):
response_one = rdb_api.add_response(
name="response_one",
values_ref=add_blob([11.1, 11.2, 9.9, 9.3]),
realization_index=realization.index,
ensemble_name=ens.name,
)
rdb_api._add_misfit(200, obs_res_def_link.id, response_one.id)
rdb_api.add_response(
name="response_two",
values_ref=add_blob([12.1, 12.2, 11.1, 11.2, 9.9, 9.3]),
realization_index=realization.index,
ensemble_name=ens.name,
)
rdb_api.add_parameter("A", "G", add_blob(1), realization.index, "ensemble_name")
rdb_api.add_parameter("B", "G", add_blob(2), realization.index, "ensemble_name")
rdb_api.add_parameter(
"key1", "group", add_blob(2), realization.index, "ensemble_name"
)
add_data(realization_0, response_def=response_definition_one, ens=ensemble)
add_data(realization_1, response_def=response_definition_one, ens=ensemble)
######## add blob #########
data_blob = add_blob([0, 1, 2, 3])
db_lookup["data_blob"] = data_blob
rdb_session.commit()
blob_session.commit()
rdb_session.close()
blob_session.close()
yield db_lookup