def post_ensemble_results(ensemble_id: str) -> None: server = ServerMonitor.get_instance() ert = ERT.enkf_facade observations = _get_from_server( f"{server.fetch_url()}/ensembles/{ensemble_id}/observations", ).json() for record in create_response_records( ert, ert.get_current_case_name(), observations ): realizations = record["data"] name = record["name"] for index, data in realizations.items(): _post_to_server( f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{name}/matrix", params={"realization_index": index, "record_class": "response"}, data=data.to_csv().encode(), headers={"content-type": "application/x-dataframe"}, ) if record["observations"] is not None: _post_to_server( f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{name}/observations", params={"realization_index": index}, json=record["observations"], )
def _get_from_server(url, headers={}, status_code=200) -> requests.Response: server = ServerMonitor.get_instance() headers["Token"] = server.fetch_auth()[1] resp = requests.get( f"{server.fetch_url()}/{url}", headers=headers, ) if resp.status_code != status_code: logger.error(f"Failed to fetch from {url}. Response: {resp.text}") return resp
def start_ert_server(): monitor = None if FeatureToggling.is_enabled("new-storage"): from ert_shared.storage.server_monitor import ServerMonitor monitor = ServerMonitor.get_instance() monitor.start() try: yield finally: if monitor is not None: monitor.shutdown()
def post_ensemble_data( ert: "LibresFacade", ensemble_size: int, update_id: Optional[str] = None, ) -> str: server = ServerMonitor.get_instance() if update_id is None: exp_response = _post_to_server( f"{server.fetch_url()}/experiments", json=create_experiment(ert), ).json() experiment_id = exp_response["id"] for obs in create_observations(ert): _post_to_server( f"{server.fetch_url()}/experiments/{experiment_id}/observations", json=obs, ) else: update = _get_from_server( f"{server.fetch_url()}/updates/{update_id}").json() experiment_id = update["experiment_id"] parameters = create_parameters(ert) response_names = [ key.split("@")[0] if ert.is_gen_data_key(key) else key for key in ert.all_data_type_keys() if ert.is_gen_data_key(key) or ert.is_summary_key(key) ] ens_response = _post_to_server( f"{server.fetch_url()}/experiments/{experiment_id}/ensembles", json=create_ensemble( ert, size=ensemble_size, parameter_names=[param["name"] for param in parameters], response_names=response_names, update_id=update_id, ), ) ensemble_id = ens_response.json()["id"] for param in parameters: df = pd.DataFrame([p.tolist() for p in param["values"]]) _post_to_server( f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{param['name']}/matrix", data=df.to_csv(), headers={"content-type": "text/csv"}, ) return ensemble_id
def post_ensemble_data(reference: Optional[Tuple[int, str]] = None) -> int: server = ServerMonitor.get_instance() ert = ERT.enkf_facade response = requests.post( f"{server.fetch_url()}/ensembles", data=create_ensemble(ert, reference).json(), auth=server.fetch_auth(), ) if not response.status_code == 200: raise RuntimeError(response.text) ens = js.Ensemble.parse_obj(response.json()) return ens.id
def post_ensemble_results(ensemble_id: int): server = ServerMonitor.get_instance() ert = ERT.enkf_facade for r in create_responses(ert, ert.get_current_case_name()): requests.post( f"{server.fetch_url()}/ensembles/{ensemble_id}/responses", data=r.json(), auth=server.fetch_auth(), ) for u in create_update_data(ert): requests.post( f"{server.fetch_url()}/ensembles/{ensemble_id}/misfit", data=u.json(), auth=server.fetch_auth(), )
def _post_to_server(url, data=None, params=None, json=None, headers={}, status_code=200) -> requests.Response: server = ServerMonitor.get_instance() headers["Token"] = server.fetch_auth()[1] resp = requests.post( f"{server.fetch_url()}/{url}", headers=headers, params=params, data=data, json=json, ) if resp.status_code != status_code: logger.error(f"Failed to post to {url}. Response: {resp.text}") return resp
def post_ensemble_data(ensemble_size: int, update_id: str = None) -> str: server = ServerMonitor.get_instance() ert = ERT.enkf_facade if update_id is None: exp_response = _post_to_server( f"{server.fetch_url()}/experiments", json=create_experiment(ert), ).json() experiment_id = exp_response["id"] for obs in create_observations(ert): _post_to_server( f"{server.fetch_url()}/experiments/{experiment_id}/observations", json=obs, ) else: update = _get_from_server(f"{server.fetch_url()}/updates/{update_id}").json() experiment_id = update["experiment_id"] parameters = create_parameters(ert) ens_response = _post_to_server( f"{server.fetch_url()}/experiments/{experiment_id}/ensembles", json=create_ensemble( ert, size=ensemble_size, parameter_names=[param["name"] for param in parameters], update_id=update_id, ), ) ensemble_id = ens_response.json()["id"] for param in parameters: _post_to_server( f"{server.fetch_url()}/ensembles/{ensemble_id}/records/{param['name']}/matrix", json=[p.tolist() for p in param["values"]], ) return ensemble_id
def post_update_data(ert: "LibresFacade", parent_ensemble_id: str, algorithm: str) -> str: server = ServerMonitor.get_instance() observations = _get_from_server( f"{server.fetch_url()}/ensembles/{parent_ensemble_id}/observations", ).json() # create update thingy update_create = dict( observation_transformations=_create_observation_transformation( ert, observations), ensemble_reference_id=parent_ensemble_id, ensemble_result_id=None, algorithm=algorithm, ) response = _post_to_server( f"{server.fetch_url()}/updates", json=update_create, ) update = response.json() return update["id"]
def post_update_data(parent_ensemble_id: int, algorithm: str) -> int: server = ServerMonitor.get_instance() ert = ERT.enkf_facade observations = requests.get( f"{server.fetch_url()}/ensembles/{parent_ensemble_id}/observations", ).json() # create update thingy update_create = dict( observation_transformations=_create_observation_transformation( ert, observations), ensemble_reference_id=parent_ensemble_id, ensemble_result_id=None, algorithm=algorithm, ) response = requests.post( f"{server.fetch_url()}/updates", json=update_create, ) update = response.json() return update["id"]
def create_client(): server = ServerMonitor.get_instance() return StorageClient(server.fetch_url(), server.fetch_auth())
def start_ert_server(): from ert_shared.storage.server_monitor import ServerMonitor monitor = ServerMonitor.get_instance() monitor.start()