def server(request, monkeypatch, tmp_path: Path):
marker = request.node.get_closest_marker("script")
if marker is None:
return
script = (
dedent(
"""\
#!/usr/bin/python3
import os
import sys
import time
fd = os.environ.get("ERT_COMM_FD")
if fd is not None: fd = int(fd)
"""
)
+ marker.args[0]
)
path = tmp_path / "script"
path.write_text(script)
path.chmod(0o755)
monkeypatch.setattr(ServerMonitor, "EXEC_ARGS", [str(path)])
monkeypatch.setattr(ServerMonitor, "TIMEOUT", 5)
proc = ServerMonitor()
proc.start()
yield proc
def server(db_apis, request):
proc = ServerMonitor(rdb_url=ERT_STORAGE.rdb_url,
blob_url=ERT_STORAGE.blob_url,
lockfile=False)
proc.start()
request.addfinalizer(lambda: proc.shutdown())
yield proc
def test_integration(request, tmpdir):
"""Actually start the server, wait for it to be online and do a health check"""
with tmpdir.as_cwd():
server = ServerMonitor()
server.start()
request.addfinalizer(lambda: server.shutdown())
resp = requests.get(
f"{server.fetch_url()}/healthcheck", auth=server.fetch_auth()
)
assert "ALL OK!" in resp.json()
# Use global connection info
conn_info = connection.get_info()
requests.get(conn_info["baseurl"] + "/healthcheck", auth=conn_info["auth"])
# Use local connection info
conn_info = connection.get_info(str(tmpdir))
requests.get(conn_info["baseurl"] + "/healthcheck", auth=conn_info["auth"])
server.shutdown()
assert not (tmpdir / "storage_server.json").exists()
# Global connection info no longer valid
with pytest.raises(RuntimeError):
connection.get_info()
def post_ensemble_results(ensemble_id: str) -> None:
server = ServerMonitor.get_instance()
ert = ERT.enkf_facade
observations = _get_from_server(
f"{server.fetch_url()}/ensembles/{ensemble_id}/observations",
).json()
for record in create_response_records(
ert, ert.get_current_case_name(), observations
):
realizations = record["data"]
name = record["name"]
for index, data in realizations.items():
_post_to_server(
f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{name}/matrix",
params={"realization_index": index, "record_class": "response"},
data=data.to_csv().encode(),
headers={"content-type": "application/x-dataframe"},
)
if record["observations"] is not None:
_post_to_server(
f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{name}/observations",
params={"realization_index": index},
json=record["observations"],
)
def test_json_exists(tmpdir):
with tmpdir.as_cwd():
with open(str(tmpdir / "storage_server.json"), "w") as f:
f.write("this is a json file")
with pytest.raises(SystemExit):
ServerMonitor()
def test_integration_auth(request, tmpdir):
"""Start the server, wait for it to be online and then do a health check with an
invalid auth"""
with tmpdir.as_cwd():
server = ServerMonitor()
server.start()
request.addfinalizer(lambda: server.shutdown())
# No auth
resp = requests.get(f"{server.fetch_url()}/healthcheck")
assert resp.status_code == 401
# Invalid auth
resp = requests.get(f"{server.fetch_url()}/healthcheck",
auth=("__token__", "invalid-token"))
assert resp.status_code == 403
def _get_from_server(url, headers={}, status_code=200) -> requests.Response:
server = ServerMonitor.get_instance()
headers["Token"] = server.fetch_auth()[1]
resp = requests.get(
f"{server.fetch_url()}/{url}",
headers=headers,
)
if resp.status_code != status_code:
logger.error(f"Failed to fetch from {url}. Response: {resp.text}")
return resp
def start_ert_server():
monitor = None
if FeatureToggling.is_enabled("new-storage"):
from ert_shared.storage.server_monitor import ServerMonitor
monitor = ServerMonitor.get_instance()
monitor.start()
try:
yield
finally:
if monitor is not None:
monitor.shutdown()
def post_ensemble_data(
ert: "LibresFacade",
ensemble_size: int,
update_id: Optional[str] = None,
) -> str:
server = ServerMonitor.get_instance()
if update_id is None:
exp_response = _post_to_server(
f"{server.fetch_url()}/experiments",
json=create_experiment(ert),
).json()
experiment_id = exp_response["id"]
for obs in create_observations(ert):
_post_to_server(
f"{server.fetch_url()}/experiments/{experiment_id}/observations",
json=obs,
)
else:
update = _get_from_server(
f"{server.fetch_url()}/updates/{update_id}").json()
experiment_id = update["experiment_id"]
parameters = create_parameters(ert)
response_names = [
key.split("@")[0] if ert.is_gen_data_key(key) else key
for key in ert.all_data_type_keys()
if ert.is_gen_data_key(key) or ert.is_summary_key(key)
]
ens_response = _post_to_server(
f"{server.fetch_url()}/experiments/{experiment_id}/ensembles",
json=create_ensemble(
ert,
size=ensemble_size,
parameter_names=[param["name"] for param in parameters],
response_names=response_names,
update_id=update_id,
),
)
ensemble_id = ens_response.json()["id"]
for param in parameters:
df = pd.DataFrame([p.tolist() for p in param["values"]])
_post_to_server(
f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{param['name']}/matrix",
data=df.to_csv(),
headers={"content-type": "text/csv"},
)
return ensemble_id
def post_ensemble_data(reference: Optional[Tuple[int, str]] = None) -> int:
server = ServerMonitor.get_instance()
ert = ERT.enkf_facade
response = requests.post(
f"{server.fetch_url()}/ensembles",
data=create_ensemble(ert, reference).json(),
auth=server.fetch_auth(),
)
if not response.status_code == 200:
raise RuntimeError(response.text)
ens = js.Ensemble.parse_obj(response.json())
return ens.id
def post_ensemble_results(ensemble_id: int):
server = ServerMonitor.get_instance()
ert = ERT.enkf_facade
for r in create_responses(ert, ert.get_current_case_name()):
requests.post(
f"{server.fetch_url()}/ensembles/{ensemble_id}/responses",
data=r.json(),
auth=server.fetch_auth(),
)
for u in create_update_data(ert):
requests.post(
f"{server.fetch_url()}/ensembles/{ensemble_id}/misfit",
data=u.json(),
auth=server.fetch_auth(),
)
def _post_to_server(url,
data=None,
params=None,
json=None,
headers={},
status_code=200) -> requests.Response:
server = ServerMonitor.get_instance()
headers["Token"] = server.fetch_auth()[1]
resp = requests.post(
f"{server.fetch_url()}/{url}",
headers=headers,
params=params,
data=data,
json=json,
)
if resp.status_code != status_code:
logger.error(f"Failed to post to {url}. Response: {resp.text}")
return resp
def post_update_data(ert: "LibresFacade", parent_ensemble_id: str,
algorithm: str) -> str:
server = ServerMonitor.get_instance()
observations = _get_from_server(
f"{server.fetch_url()}/ensembles/{parent_ensemble_id}/observations",
).json()
# create update thingy
update_create = dict(
observation_transformations=_create_observation_transformation(
ert, observations),
ensemble_reference_id=parent_ensemble_id,
ensemble_result_id=None,
algorithm=algorithm,
)
response = _post_to_server(
f"{server.fetch_url()}/updates",
json=update_create,
)
update = response.json()
return update["id"]
def post_ensemble_data(ensemble_size: int, update_id: str = None) -> str:
server = ServerMonitor.get_instance()
ert = ERT.enkf_facade
if update_id is None:
exp_response = _post_to_server(
f"{server.fetch_url()}/experiments",
json=create_experiment(ert),
).json()
experiment_id = exp_response["id"]
for obs in create_observations(ert):
_post_to_server(
f"{server.fetch_url()}/experiments/{experiment_id}/observations",
json=obs,
)
else:
update = _get_from_server(f"{server.fetch_url()}/updates/{update_id}").json()
experiment_id = update["experiment_id"]
parameters = create_parameters(ert)
ens_response = _post_to_server(
f"{server.fetch_url()}/experiments/{experiment_id}/ensembles",
json=create_ensemble(
ert,
size=ensemble_size,
parameter_names=[param["name"] for param in parameters],
update_id=update_id,
),
)
ensemble_id = ens_response.json()["id"]
for param in parameters:
_post_to_server(
f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{param['name']}/matrix",
json=[p.tolist() for p in param["values"]],
)
return ensemble_id
def post_update_data(parent_ensemble_id: int, algorithm: str) -> int:
server = ServerMonitor.get_instance()
ert = ERT.enkf_facade
observations = requests.get(
f"{server.fetch_url()}/ensembles/{parent_ensemble_id}/observations",
).json()
# create update thingy
update_create = dict(
observation_transformations=_create_observation_transformation(
ert, observations),
ensemble_reference_id=parent_ensemble_id,
ensemble_result_id=None,
algorithm=algorithm,
)
response = requests.post(
f"{server.fetch_url()}/updates",
json=update_create,
)
update = response.json()
return update["id"]
def server(db_api, request):
proc = ServerMonitor(rdb_url=ERT_STORAGE.SQLALCHEMY_URL, lockfile=False)
proc.start()
request.addfinalizer(lambda: proc.shutdown())
yield proc
def create_client():
server = ServerMonitor.get_instance()
return StorageClient(server.fetch_url(), server.fetch_auth())
def start_ert_server():
from ert_shared.storage.server_monitor import ServerMonitor
monitor = ServerMonitor.get_instance()
monitor.start()