async def post_record_observations(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
observation_ids: List[UUID] = Body(...),
) -> None:
if realization_index is None:
record_obj = _get_ensemble_record(db, ensemble_id, name)
else:
record_obj = _get_forward_model_record(db, ensemble_id, name,
realization_index)
observations = (db.query(ds.Observation).filter(
ds.Observation.id.in_(observation_ids)).all())
if observations:
record_obj.observations = observations
flag_modified(record_obj, "observations")
db.commit()
else:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail={
"error": f"Observations {observation_ids} not found!",
"name": name,
"ensemble_id": str(ensemble_id),
},
)
async def get_ensemble_response_dataframe(*,
db: Session = Depends(get_db),
ensemble_id: UUID,
response_name: str) -> Response:
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
records = (db.query(ds.Record).filter(
ds.Record.realization_index != None).join(ds.RecordInfo).filter_by(
ensemble_pk=ensemble.pk,
name=response_name,
record_class=ds.RecordClass.response,
)).all()
df_list = []
for record in records:
data_df = pd.DataFrame(record.f64_matrix.content)
labels = record.f64_matrix.labels
if labels is not None:
# if the realization is more than 1D array
# the next line will produce ValueError exception
data_df.index = [record.realization_index]
data_df.columns = labels[0]
df_list.append(data_df)
return Response(
content=pd.concat(df_list, axis=0).to_csv().encode(),
media_type="text/csv",
)
def get_record_by_name(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> ds.Record:
try:
return (
db.query(ds.Record)
.filter_by(realization_index=realization_index)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
.one()
)
except NoResultFound as e:
pass
if realization_index is not None:
return (
db.query(ds.Record)
.filter_by(realization_index=None)
.join(ds.RecordInfo)
.filter_by(
name=name,
record_type=ds.RecordType.f64_matrix,
)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
.one()
)
raise exc.NotFoundError(f"Record not found")
async def create_blob(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> None:
"""
Create a record which points to a blob on Azure Blob Storage.
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
file_obj = ds.File(
filename="test",
mimetype="mime/type",
)
if HAS_AZURE_BLOB_STORAGE:
key = f"{name}@{realization_index}@{uuid4()}"
blob = azure_blob_container.get_blob_client(key)
file_obj.az_container = (azure_blob_container.container_name, )
file_obj.az_blob = (key, )
else:
pass
db.add(file_obj)
_create_record(
db,
ensemble,
name,
ds.RecordType.file,
realization_index=realization_index,
file=file_obj,
)
async def finalize_blob(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> None:
"""
Commit all staged blocks to a blob record
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
record_obj = (db.query(
ds.Record).filter_by(realization_index=realization_index).join(
ds.RecordInfo).filter_by(ensemble_pk=ensemble.pk, name=name).one())
submitted_blocks = list(
db.query(ds.FileBlock).filter_by(
record_name=name,
ensemble_pk=ensemble.pk,
realization_index=realization_index,
).all())
if HAS_AZURE_BLOB_STORAGE:
key = record_obj.file.az_blob
blob = azure_blob_container.get_blob_client(key)
block_ids = [
block.block_id
for block in sorted(submitted_blocks, key=lambda x: x.block_index)
]
await blob.commit_block_list(block_ids)
else:
data = b"".join([block.content for block in submitted_blocks])
record_obj.file.content = data
async def replace_record_userdata(
*,
db: Session = Depends(get_db),
record: ds.Record = Depends(get_record_by_name),
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
record.userdata = body
db.commit()
async def replace_observation_userdata(
*,
db: Session = Depends(get_db),
obs_id: UUID,
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
obs = db.query(ds.Observation).filter_by(id=obs_id).one()
obs.userdata = body
db.commit()
async def add_block(
*,
db: Session = Depends(get_db),
bh: BlobHandler = Depends(get_blob_handler),
record: ds.Record = Depends(get_record_by_name),
request: Request,
block_index: int,
) -> None:
"""
Stage blocks to an existing azure blob record.
"""
db.add(await bh.stage_blob(record, request, block_index))
async def replace_experiment_userdata(
*,
db: Session = Depends(get_db),
experiment_id: UUID,
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
experiment.userdata = body
db.commit()
async def patch_record_userdata(
*,
db: Session = Depends(get_db),
record: ds.Record = Depends(get_record_by_name),
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
record.userdata.update(body)
flag_modified(record, "userdata")
db.commit()
async def replace_ensemble_userdata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
ensemble.userdata = body
db.commit()
async def patch_observation_userdata(
*,
db: Session = Depends(get_db),
obs_id: UUID,
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
obs = db.query(ds.Observation).filter_by(id=obs_id).one()
obs.userdata.update(body)
flag_modified(obs, "userdata")
db.commit()
async def patch_experiment_userdata(
*,
db: Session = Depends(get_db),
experiment_id: UUID,
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
experiment.userdata.update(body)
flag_modified(experiment, "userdata")
db.commit()
async def patch_ensemble_userdata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
ensemble.userdata.update(body)
flag_modified(ensemble, "userdata")
db.commit()
async def post_record_observations(
*,
db: Session = Depends(get_db),
record: ds.Record = Depends(get_record_by_name),
observation_ids: List[UUID] = Body(...),
) -> None:
observations = (
db.query(ds.Observation).filter(ds.Observation.id.in_(observation_ids)).all()
)
if observations:
record.observations = observations
db.commit()
else:
raise exc.UnprocessableError(f"Observations {observation_ids} not found!")
async def get_record_labels(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
) -> List[str]:
"""
Get the list of record data labels. If the record is not a group record the list of labels will
contain only the name of the record
Example
- Group record:
data - {"a": 4, "b":42, "c": 2}
return: ["a", "b", "c"]
- Ensemble record:
data - [4, 42, 2, 32]
return: []
"""
record = (
db.query(ds.Record)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
.first()
)
if record is None:
raise exc.NotFoundError(f"Record not found")
if record.f64_matrix and record.f64_matrix.labels:
return record.f64_matrix.labels[0]
return []
def get_update(
*,
db: Session = Depends(get_db),
update_id: UUID,
) -> js.UpdateOut:
update_obj = db.query(ds.Update).filter_by(id=update_id).one()
return _update_from_db(update_obj)
def get_ensemble_responses(*, db: Session = Depends(get_db),
ensemble_id: UUID) -> Mapping[str, ds.Record]:
return {
rec.name: rec
for rec in (db.query(ds.Record).join(ds.RecordInfo).filter_by(
record_class=ds.RecordClass.response).join(ds.Ensemble).filter_by(
id=ensemble_id).all())
}
async def replace_record_metadata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
body: Any = Body(...),
) -> None:
"""
Assign new metadata json
"""
if realization_index is None:
record_obj = _get_ensemble_record(db, ensemble_id, name)
else:
record_obj = _get_forward_model_record(db, ensemble_id, name,
realization_index)
record_obj._metadata = body
db.commit()
def _create_record(
db: Session,
record: ds.Record,
) -> ds.Record:
nested = db.begin_nested()
try:
db.add(record)
db.commit()
except IntegrityError:
# Assuming this is a UNIQUE constraint failure due to an existing
# record_info with the same name and ensemble. Try to fetch the
# record_info
nested.rollback()
record_info = record.record_info
old_record_info = (
db.query(ds.RecordInfo)
.filter_by(ensemble=record_info.ensemble, name=record_info.name)
.one()
)
# Check that the parameters match
if record_info.record_class != old_record_info.record_class:
raise exc.ConflictError(
"Record class of new record does not match previous record class",
new_record_class=record_info.record_class,
old_record_class=old_record_info.record_class,
)
if record_info.record_type != old_record_info.record_type:
raise exc.ConflictError(
"Record type of new record does not match previous record type",
new_record_type=record_info.record_type,
old_record_type=old_record_info.record_type,
)
record = ds.Record(
record_info=old_record_info,
f64_matrix=record.f64_matrix,
file=record.file,
realization_index=record.realization_index,
)
db.add(record)
db.commit()
return record
def execute(self, *args: Any, **kwargs: Any) -> ExecutionResult:
kwargs.setdefault("context_value", {})
if self.override_session is not None:
session_obj = self.override_session()
else:
session_obj = Session()
with session_obj as session:
kwargs["context_value"]["session"] = session
return super().execute(*args, **kwargs)
async def get_observation_userdata(
*,
db: Session = Depends(get_db),
obs_id: UUID,
) -> Mapping[str, Any]:
"""
Get userdata json
"""
obs = db.query(ds.Observation).filter_by(id=obs_id).one()
return obs.userdata
async def get_ensemble_userdata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
) -> Mapping[str, Any]:
"""
Get userdata json
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
return ensemble.userdata
async def get_experiment_userdata(
*,
db: Session = Depends(get_db),
experiment_id: UUID,
) -> Mapping[str, Any]:
"""
Get userdata json
"""
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
return experiment.userdata
def _get_forward_model_record(db: Session, ensemble_id: UUID, name: str,
realization_index: int) -> ds.Record:
try:
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
return (db.query(ds.Record).filter_by(
realization_index=realization_index).join(ds.RecordInfo).filter_by(
ensemble_pk=ensemble.pk,
name=name,
).one())
except NoResultFound:
raise HTTPException(
status_code=404,
detail={
"error":
f"Forward-model record '{name}' for ensemble '{ensemble_id}', realization {realization_index} not found!",
"name": name,
"ensemble_id": str(ensemble_id),
},
)
async def get_response_misfits(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
response_name: str,
realization_index: Optional[int] = None,
summary_misfits: bool = False,
) -> Response:
"""
Compute univariate misfits for response(s)
"""
response_query = (db.query(ds.Record).filter(
ds.Record.observations != None).join(ds.RecordInfo).filter_by(
name=response_name,
record_type=ds.RecordType.f64_matrix,
).join(ds.Ensemble).filter_by(id=ensemble_id))
if realization_index is not None:
responses = [
response_query.filter(
ds.Record.realization_index == realization_index).one()
]
else:
responses = response_query.all()
observation_df = None
response_dict = {}
for response in responses:
data_df = pd.DataFrame(response.f64_matrix.content)
labels = response.f64_matrix.labels
if labels is not None:
data_df.columns = labels[0]
data_df.index = labels[1]
response_dict[response.realization_index] = data_df
if observation_df is None:
# currently we expect only a single observation object, while
# later in the future this might change
obs = response.observations[0]
observation_df = pd.DataFrame(data={
"values": obs.values,
"errors": obs.errors
},
index=obs.x_axis)
try:
result_df = calculate_misfits_from_pandas(response_dict,
observation_df,
summary_misfits)
except Exception as misfits_exc:
raise exc.UnprocessableError(
f"Unable to compute misfits: {misfits_exc}")
return Response(
content=result_df.to_csv().encode(),
media_type="text/csv",
)
def new_record(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> ds.Record:
"""
Get ensemble and verify that no record with the name `name` exists
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
q = (
db.query(ds.Record)
.join(ds.RecordInfo)
.filter_by(ensemble_pk=ensemble.pk, name=name)
)
if (
ensemble.size != -1
and realization_index is not None
and realization_index not in ensemble.active_realizations
):
raise exc.ExpectationError(
f"Realization index {realization_index} outside of allowed realization indices {ensemble.active_realizations}"
)
q = q.filter(
(ds.Record.realization_index == None)
| (ds.Record.realization_index == realization_index)
)
if q.count() > 0:
raise exc.ConflictError(
f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' already exists",
)
return ds.Record(
record_info=ds.RecordInfo(
ensemble=ensemble,
name=name,
),
realization_index=realization_index,
)
def get_records_by_name(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> List[ds.Record]:
records = (
db.query(ds.Record)
.filter_by(realization_index=realization_index)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
).all()
if not records:
records = (
db.query(ds.Record)
.join(ds.RecordInfo)
.filter_by(
name=name,
record_type=ds.RecordType.f64_matrix,
)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
).all()
if not records:
records = (
db.query(ds.Record)
.filter_by(realization_index=None)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
).all()
if not records:
raise exc.NotFoundError(f"Record not found")
return records
def _get_and_assert_ensemble(db: Session, ensemble_id: UUID, name: str,
realization_index: Optional[int]) -> ds.Ensemble:
"""
Get ensemble and verify that no record with the name `name` exists
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
q = (db.query(ds.Record).join(ds.RecordInfo).filter_by(
ensemble_pk=ensemble.pk, name=name))
if realization_index is not None:
if realization_index not in range(
ensemble.size) and ensemble.size != -1:
raise HTTPException(
status_code=status.HTTP_417_EXPECTATION_FAILED,
detail={
"error":
f"Ensemble '{name}' ('{ensemble_id}') does have a 'size' "
f"of {ensemble.size}. The posted record is targeting "
f"'realization_index' {realization_index} which is out "
f"of bounds.",
"name":
name,
"ensemble_id":
str(ensemble_id),
},
)
q = q.filter((ds.Record.realization_index == None)
| (ds.Record.realization_index == realization_index))
if q.count() > 0:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"error":
f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' already exists",
"name": name,
"ensemble_id": str(ensemble_id),
},
)
return ensemble
async def patch_record_metadata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
body: Any = Body(...),
) -> None:
"""
Update metadata json
"""
if realization_index is None:
record_obj = _get_ensemble_record(db, ensemble_id, name)
else:
record_obj = _get_forward_model_record(db, ensemble_id, name,
realization_index)
record_obj._metadata.update(body)
flag_modified(record_obj, "_metadata")
db.commit()
