async def finalize_blob(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> None:
"""
Commit all staged blocks to a blob record
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
record_obj = (db.query(
ds.Record).filter_by(realization_index=realization_index).join(
ds.RecordInfo).filter_by(ensemble_pk=ensemble.pk, name=name).one())
submitted_blocks = list(
db.query(ds.FileBlock).filter_by(
record_name=name,
ensemble_pk=ensemble.pk,
realization_index=realization_index,
).all())
if HAS_AZURE_BLOB_STORAGE:
key = record_obj.file.az_blob
blob = azure_blob_container.get_blob_client(key)
block_ids = [
block.block_id
for block in sorted(submitted_blocks, key=lambda x: x.block_index)
]
await blob.commit_block_list(block_ids)
else:
data = b"".join([block.content for block in submitted_blocks])
record_obj.file.content = data
async def get_ensemble_response_dataframe(*,
db: Session = Depends(get_db),
ensemble_id: UUID,
response_name: str) -> Response:
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
records = (db.query(ds.Record).filter(
ds.Record.realization_index != None).join(ds.RecordInfo).filter_by(
ensemble_pk=ensemble.pk,
name=response_name,
record_class=ds.RecordClass.response,
)).all()
df_list = []
for record in records:
data_df = pd.DataFrame(record.f64_matrix.content)
labels = record.f64_matrix.labels
if labels is not None:
# if the realization is more than 1D array
# the next line will produce ValueError exception
data_df.index = [record.realization_index]
data_df.columns = labels[0]
df_list.append(data_df)
return Response(
content=pd.concat(df_list, axis=0).to_csv().encode(),
media_type="text/csv",
)
def post_ensemble(*,
db: Session = Depends(get_db),
ens_in: js.EnsembleIn,
experiment_id: UUID) -> ds.Ensemble:
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
active_reals = (ens_in.active_realizations if ens_in.active_realizations
else list(range(ens_in.size)))
if ens_in.size > 0:
if max(active_reals) > ens_in.size - 1:
raise exc.ExpectationError(
f"Ensemble active realization index {max(active_reals)} out of realization range [0,{ ens_in.size - 1}]"
)
if len(set(active_reals)) != len(active_reals):
raise exc.ExpectationError(
f"Non unique active realization index list not allowed {active_reals}"
)
ens = ds.Ensemble(
parameter_names=ens_in.parameter_names,
response_names=ens_in.response_names,
experiment=experiment,
size=ens_in.size,
userdata=ens_in.userdata,
active_realizations=active_reals,
)
db.add(ens)
if ens_in.update_id:
update_obj = db.query(ds.Update).filter_by(id=ens_in.update_id).one()
update_obj.ensemble_result = ens
db.commit()
return ens
def get_record_by_name(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> ds.Record:
try:
return (
db.query(ds.Record)
.filter_by(realization_index=realization_index)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
.one()
)
except NoResultFound as e:
pass
if realization_index is not None:
return (
db.query(ds.Record)
.filter_by(realization_index=None)
.join(ds.RecordInfo)
.filter_by(
name=name,
record_type=ds.RecordType.f64_matrix,
)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
.one()
)
raise exc.NotFoundError(f"Record not found")
def get_update(
*,
db: Session = Depends(get_db),
update_id: UUID,
) -> js.UpdateOut:
update_obj = db.query(ds.Update).filter_by(id=update_id).one()
return _update_from_db(update_obj)
async def create_blob(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> None:
"""
Create a record which points to a blob on Azure Blob Storage.
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
file_obj = ds.File(
filename="test",
mimetype="mime/type",
)
if HAS_AZURE_BLOB_STORAGE:
key = f"{name}@{realization_index}@{uuid4()}"
blob = azure_blob_container.get_blob_client(key)
file_obj.az_container = (azure_blob_container.container_name, )
file_obj.az_blob = (key, )
else:
pass
db.add(file_obj)
_create_record(
db,
ensemble,
name,
ds.RecordType.file,
realization_index=realization_index,
file=file_obj,
)
async def post_record_observations(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
observation_ids: List[UUID] = Body(...),
) -> None:
if realization_index is None:
record_obj = _get_ensemble_record(db, ensemble_id, name)
else:
record_obj = _get_forward_model_record(db, ensemble_id, name,
realization_index)
observations = (db.query(ds.Observation).filter(
ds.Observation.id.in_(observation_ids)).all())
if observations:
record_obj.observations = observations
flag_modified(record_obj, "observations")
db.commit()
else:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail={
"error": f"Observations {observation_ids} not found!",
"name": name,
"ensemble_id": str(ensemble_id),
},
)
async def get_record_labels(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
) -> List[str]:
"""
Get the list of record data labels. If the record is not a group record the list of labels will
contain only the name of the record
Example
- Group record:
data - {"a": 4, "b":42, "c": 2}
return: ["a", "b", "c"]
- Ensemble record:
data - [4, 42, 2, 32]
return: []
"""
record = (
db.query(ds.Record)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
.first()
)
if record is None:
raise exc.NotFoundError(f"Record not found")
if record.f64_matrix and record.f64_matrix.labels:
return record.f64_matrix.labels[0]
return []
def get_ensemble_responses(*, db: Session = Depends(get_db),
ensemble_id: UUID) -> Mapping[str, ds.Record]:
return {
rec.name: rec
for rec in (db.query(ds.Record).join(ds.RecordInfo).filter_by(
record_class=ds.RecordClass.response).join(ds.Ensemble).filter_by(
id=ensemble_id).all())
}
async def get_observation_userdata(
*,
db: Session = Depends(get_db),
obs_id: UUID,
) -> Mapping[str, Any]:
"""
Get userdata json
"""
obs = db.query(ds.Observation).filter_by(id=obs_id).one()
return obs.userdata
def _get_forward_model_record(db: Session, ensemble_id: UUID, name: str,
realization_index: int) -> ds.Record:
try:
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
return (db.query(ds.Record).filter_by(
realization_index=realization_index).join(ds.RecordInfo).filter_by(
ensemble_pk=ensemble.pk,
name=name,
).one())
except NoResultFound:
raise HTTPException(
status_code=404,
detail={
"error":
f"Forward-model record '{name}' for ensemble '{ensemble_id}', realization {realization_index} not found!",
"name": name,
"ensemble_id": str(ensemble_id),
},
)
async def get_experiment_userdata(
*,
db: Session = Depends(get_db),
experiment_id: UUID,
) -> Mapping[str, Any]:
"""
Get userdata json
"""
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
return experiment.userdata
async def get_ensemble_userdata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
) -> Mapping[str, Any]:
"""
Get userdata json
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
return ensemble.userdata
async def get_response_misfits(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
response_name: str,
realization_index: Optional[int] = None,
summary_misfits: bool = False,
) -> Response:
"""
Compute univariate misfits for response(s)
"""
response_query = (db.query(ds.Record).filter(
ds.Record.observations != None).join(ds.RecordInfo).filter_by(
name=response_name,
record_type=ds.RecordType.f64_matrix,
).join(ds.Ensemble).filter_by(id=ensemble_id))
if realization_index is not None:
responses = [
response_query.filter(
ds.Record.realization_index == realization_index).one()
]
else:
responses = response_query.all()
observation_df = None
response_dict = {}
for response in responses:
data_df = pd.DataFrame(response.f64_matrix.content)
labels = response.f64_matrix.labels
if labels is not None:
data_df.columns = labels[0]
data_df.index = labels[1]
response_dict[response.realization_index] = data_df
if observation_df is None:
# currently we expect only a single observation object, while
# later in the future this might change
obs = response.observations[0]
observation_df = pd.DataFrame(data={
"values": obs.values,
"errors": obs.errors
},
index=obs.x_axis)
try:
result_df = calculate_misfits_from_pandas(response_dict,
observation_df,
summary_misfits)
except Exception as misfits_exc:
raise exc.UnprocessableError(
f"Unable to compute misfits: {misfits_exc}")
return Response(
content=result_df.to_csv().encode(),
media_type="text/csv",
)
def create_update(
*,
db: Session = Depends(get_db),
update: js.UpdateIn,
) -> js.UpdateOut:
ensemble = db.query(
ds.Ensemble).filter_by(id=update.ensemble_reference_id).one()
update_obj = ds.Update(
algorithm=update.algorithm,
ensemble_reference_pk=ensemble.pk,
)
db.add(update_obj)
if update.observation_transformations:
transformations = {
t.name: t
for t in update.observation_transformations
}
observation_ids = [t.observation_id for t in transformations.values()]
observations = (db.query(ds.Observation).filter(
ds.Observation.id.in_(observation_ids)).all())
observation_transformations = [
ds.ObservationTransformation(
active_list=transformations[observation.name].active,
scale_list=transformations[observation.name].scale,
observation=observation,
update=update_obj,
) for observation in observations
]
db.add_all(observation_transformations)
db.commit()
return js.UpdateOut(
id=update_obj.id,
experiment_id=ensemble.experiment.id,
algorithm=update_obj.algorithm,
ensemble_reference_id=ensemble.id,
)
def get_records_by_name(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> List[ds.Record]:
records = (
db.query(ds.Record)
.filter_by(realization_index=realization_index)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
).all()
if not records:
records = (
db.query(ds.Record)
.join(ds.RecordInfo)
.filter_by(
name=name,
record_type=ds.RecordType.f64_matrix,
)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
).all()
if not records:
records = (
db.query(ds.Record)
.filter_by(realization_index=None)
.join(ds.RecordInfo)
.filter_by(name=name)
.join(ds.Ensemble)
.filter_by(id=ensemble_id)
).all()
if not records:
raise exc.NotFoundError(f"Record not found")
return records
def new_record(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
) -> ds.Record:
"""
Get ensemble and verify that no record with the name `name` exists
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
q = (
db.query(ds.Record)
.join(ds.RecordInfo)
.filter_by(ensemble_pk=ensemble.pk, name=name)
)
if (
ensemble.size != -1
and realization_index is not None
and realization_index not in ensemble.active_realizations
):
raise exc.ExpectationError(
f"Realization index {realization_index} outside of allowed realization indices {ensemble.active_realizations}"
)
q = q.filter(
(ds.Record.realization_index == None)
| (ds.Record.realization_index == realization_index)
)
if q.count() > 0:
raise exc.ConflictError(
f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' already exists",
)
return ds.Record(
record_info=ds.RecordInfo(
ensemble=ensemble,
name=name,
),
realization_index=realization_index,
)
def _get_and_assert_ensemble(db: Session, ensemble_id: UUID, name: str,
realization_index: Optional[int]) -> ds.Ensemble:
"""
Get ensemble and verify that no record with the name `name` exists
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
q = (db.query(ds.Record).join(ds.RecordInfo).filter_by(
ensemble_pk=ensemble.pk, name=name))
if realization_index is not None:
if realization_index not in range(
ensemble.size) and ensemble.size != -1:
raise HTTPException(
status_code=status.HTTP_417_EXPECTATION_FAILED,
detail={
"error":
f"Ensemble '{name}' ('{ensemble_id}') does have a 'size' "
f"of {ensemble.size}. The posted record is targeting "
f"'realization_index' {realization_index} which is out "
f"of bounds.",
"name":
name,
"ensemble_id":
str(ensemble_id),
},
)
q = q.filter((ds.Record.realization_index == None)
| (ds.Record.realization_index == realization_index))
if q.count() > 0:
raise HTTPException(
status_code=status.HTTP_409_CONFLICT,
detail={
"error":
f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' already exists",
"name": name,
"ensemble_id": str(ensemble_id),
},
)
return ensemble
async def replace_experiment_userdata(
*,
db: Session = Depends(get_db),
experiment_id: UUID,
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
experiment.userdata = body
db.commit()
def post_ensemble(*,
db: Session = Depends(get_db),
ens_in: js.EnsembleIn,
experiment_id: UUID) -> ds.Ensemble:
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
ens = ds.Ensemble(
parameter_names=ens_in.parameter_names,
response_names=ens_in.response_names,
experiment=experiment,
size=ens_in.size,
_metadata=ens_in.metadata,
)
db.add(ens)
if ens_in.update_id:
update_obj = db.query(ds.Update).filter_by(id=ens_in.update_id).one()
update_obj.ensemble_result = ens
db.commit()
return ens
async def replace_observation_userdata(
*,
db: Session = Depends(get_db),
obs_id: UUID,
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
obs = db.query(ds.Observation).filter_by(id=obs_id).one()
obs.userdata = body
db.commit()
async def replace_ensemble_userdata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
body: Any = Body(...),
) -> None:
"""
Assign new userdata json
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
ensemble.userdata = body
db.commit()
async def patch_experiment_userdata(
*,
db: Session = Depends(get_db),
experiment_id: UUID,
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
experiment.userdata.update(body)
flag_modified(experiment, "userdata")
db.commit()
def post_observation(*,
db: Session = Depends(get_db),
obs_in: js.ObservationIn,
experiment_id: UUID) -> js.ObservationOut:
experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one()
records = ([
db.query(ds.Record).filter_by(id=rec_id).one()
for rec_id in obs_in.records
] if obs_in.records is not None else [])
obs = ds.Observation(
name=obs_in.name,
x_axis=obs_in.x_axis,
errors=obs_in.errors,
values=obs_in.values,
experiment=experiment,
records=records,
)
db.add(obs)
db.commit()
return _observation_from_db(obs)
async def patch_observation_userdata(
*,
db: Session = Depends(get_db),
obs_id: UUID,
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
obs = db.query(ds.Observation).filter_by(id=obs_id).one()
obs.userdata.update(body)
flag_modified(obs, "userdata")
db.commit()
async def patch_ensemble_userdata(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
body: Any = Body(...),
) -> None:
"""
Update userdata json
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
ensemble.userdata.update(body)
flag_modified(ensemble, "userdata")
db.commit()
async def get_record_data(
*,
db: Session = Depends(get_db),
record_id: UUID,
accept: Optional[str] = Header(default="application/json"),
) -> Any:
if accept == "application/x-dataframe":
logger.warning(
"Accept with 'application/x-dataframe' is deprecated. Use 'text/csv' instead."
)
accept = "text/csv"
record = db.query(ds.Record).filter_by(id=record_id).one()
return await _get_record_data(record, accept)
async def post_record_observations(
*,
db: Session = Depends(get_db),
record: ds.Record = Depends(get_record_by_name),
observation_ids: List[UUID] = Body(...),
) -> None:
observations = (
db.query(ds.Observation).filter(ds.Observation.id.in_(observation_ids)).all()
)
if observations:
record.observations = observations
db.commit()
else:
raise exc.UnprocessableError(f"Observations {observation_ids} not found!")
async def add_block(
*,
db: Session = Depends(get_db),
ensemble_id: UUID,
name: str,
realization_index: Optional[int] = None,
request: Request,
block_index: int,
) -> None:
"""
Stage blocks to an existing azure blob record.
"""
ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
block_id = str(uuid4())
file_block_obj = ds.FileBlock(
ensemble=ensemble,
block_id=block_id,
block_index=block_index,
record_name=name,
realization_index=realization_index,
)
record_obj = (db.query(
ds.Record).filter_by(realization_index=realization_index).join(
ds.RecordInfo).filter_by(ensemble_pk=ensemble.pk, name=name).one())
if HAS_AZURE_BLOB_STORAGE:
key = record_obj.file.az_blob
blob = azure_blob_container.get_blob_client(key)
await blob.stage_block(block_id, await request.body())
else:
file_block_obj.content = await request.body()
db.add(file_block_obj)
db.commit()
def get_observations_with_transformation(
*, db: Session = Depends(get_db),
ensemble_id: UUID) -> List[js.ObservationOut]:
ens = db.query(ds.Ensemble).filter_by(id=ensemble_id).one()
experiment = ens.experiment
update = ens.parent
transformations = ({
trans.observation.name: trans
for trans in update.observation_transformations
} if update is not None else {})
return [
_observation_from_db(obs, transformations)
for obs in experiment.observations
]
