async def read_item(*, db: Session = Db(), ensemble_id: int, index: int):
response_definitions = db.query(
ds.ResponseDefinition).filter_by(ensemble_id=ensemble_id)
responses = [{
"name":
resp_def.name,
"response":
(db.query(ds.Response).filter_by(response_definition_id=resp_def.id,
index=index).one()),
} for resp_def in response_definitions]
parameters = [{
"name": param.name,
"value": param.values[index],
} for param in db.query(ds.Parameter).filter_by(ensemble_id=ensemble_id)]
return_schema = {
"name":
index,
"responses": [{
"name": res["name"],
"data": res["response"].values
} for res in responses],
"parameters": [{
"name": par["name"],
"data": par["value"]
} for par in parameters],
}
return return_schema
async def create_observation_attributes_by_name(
*,
db: Session = Db(),
name: str,
attrs: Mapping[str, str],
):
obs = db.query(ds.Observation).filter_by(name=name).one()
for key, val in attrs.items():
obs.add_attribute(key, val)
db.commit()
async def read_observation_by_name(
*,
db: Session = Db(),
name: str,
):
return (db.query(ds.Observation).filter_by(name=name).order_by(
ds.Observation.id.desc()).one())
async def read_observation_attributes_by_name(
*,
db: Session = Db(),
name: str,
):
obs = db.query(ds.Observation).filter_by(name=name).one()
return obs.get_attributes()
async def create_observation(
*,
db: Session = Db(),
obs: js.ObservationCreate,
):
if db.query(ds.Observation).filter_by(name=obs.name).count() > 0:
return
obj = ds.Observation(
name=obs.name,
x_axis=obs.x_axis,
values=obs.values,
errors=obs.errors,
)
db.add(obj)
return obj
async def read_items(
*,
db: Session = Db(),
):
return {
"ensembles":
[_ensemble_minimal(x) for x in db.query(ds.Ensemble).all()]
}
def _ensemble(db: Session, ensemble: ds.Ensemble) -> dict:
doc = _ensemble_minimal(ensemble)
doc.update({
"realizations":
ensemble.num_realizations,
"responses": [{
"name": resp.name,
"id": resp.id
} for resp in db.query(ds.ResponseDefinition).filter_by(
ensemble_id=ensemble.id).all()],
"parameters": [
_parameter_minimal(par)
for par in db.query(ds.Parameter).filter_by(
ensemble_id=ensemble.id).all()
],
})
return doc
async def read_ensemble_by_name(
*,
db: Session = Db(),
name: str,
):
return _ensemble(
db,
(db.query(ds.Ensemble).filter_by(name=name).order_by(
ds.Ensemble.id.desc()).first()),
)
async def read_responses_as_csv(*,
db: Session = Db(),
ensemble_id: int,
id: int):
df = pd.DataFrame([
ref.values for ref in (db.query(ds.Response.values).filter(
ds.Response.response_definition_id == id).all())
])
return Response(content=df.to_csv(index=False, header=False),
media_type="text/csv")
async def parameter_data_by_id(*,
db: Session = Db(),
ensemble_id: int,
id: int):
df = pd.DataFrame([
db.query(ds.Parameter.values).filter_by(ensemble_id=ensemble_id,
id=id).one().values
]).T
return Response(content=df.to_csv(index=True, header=False),
media_type="text/csv")
async def create_misfit(
*,
db: Session = Db(),
id: int,
misfit: js.MisfitCreate,
):
observation = (db.query(
ds.Observation.id).filter_by(name=misfit.observation_key).one())
response_definition = (db.query(ds.ResponseDefinition.id).filter_by(
ensemble_id=id, name=misfit.response_definition_key).one())
ensemble = db.query(ds.Ensemble).filter_by(id=id).one()
obj = (db.query(ds.ObservationResponseDefinitionLink).filter_by(
observation_id=observation.id,
response_definition_id=response_definition.id).one())
if ensemble.parent is not None:
obj.update_id = ensemble.parent.id
obj = (db.query(ds.ObservationResponseDefinitionLink).filter_by(
observation_id=observation.id,
response_definition_id=response_definition.id).one())
db.flush()
responses = ((response, misfit.realizations[response.index])
for response in (db.query(ds.Response).filter(
ds.Response.index.in_(misfit.realizations.keys())).join(
ds.Response.response_definition).filter(
ds.ResponseDefinition.ensemble_id == id).all()))
db.add_all(
ds.Misfit(
observation_response_definition_link=obj,
response=response,
value=value,
) for response, value in responses)
async def read_observation_data(
*,
db: Session = Db(),
id: int,
):
data = db.query(ds.Observation).filter_by(id=id).one()
df = pd.DataFrame(
data={
"values": data.values,
"errors": data.errors,
},
index=data.key_indices,
)
return Response(content=df.to_csv(), media_type="text/csv")
async def create_responses(*,
db: Session = Db(),
ensemble_id: int,
resp: js.ResponseCreate):
exists = (db.query(ds.ResponseDefinition.id).filter_by(
ensemble_id=ensemble_id, name=resp.name).count() > 0)
if not exists:
obj = ds.ResponseDefinition(name=resp.name,
indices=resp.indices,
ensemble_id=ensemble_id)
db.add(obj)
else:
obj = (db.query(ds.ResponseDefinition).filter_by(
ensemble_id=ensemble_id, name=resp.name).one())
obj.indices = resp.indices
db.add_all(
ds.Response(values=values, index=index, response_definition=obj)
for index, values in resp.realizations.items())
db.commit()
async def read_observations(*, db: Session = Db()):
return db.query(ds.Observation).all()
async def read_observation_attributes(
*,
db: Session = Db(),
id: int,
):
return db.query(ds.ObservationAttribute).filter_by(observation_id=id).all()
async def read_observation_by_id(
*,
db: Session = Db(),
id: int,
):
return db.query(ds.Observation).filter_by(id=id).one()
async def read_ensemble_by_id(
*,
db: Session = Db(),
id: int,
):
return _ensemble(db, (db.query(ds.Ensemble).filter_by(id=id).one()))
async def parameter_by_id(*, db: Session = Db(), ensemble_id: int, id: int):
return (db.query(ds.Parameter).join(
ds.Parameter.prior, isouter=True).filter(
ds.Parameter.id == id,
ds.Parameter.ensemble_id == ensemble_id,
).one())
async def read_response_by_id(*, db: Session = Db(), ensemble_id: int,
id: int):
bundle = (db.query(ds.ResponseDefinition).filter(
ds.ResponseDefinition.id == id,
ds.ResponseDefinition.ensemble_id == ensemble_id,
).one())
observation_links = bundle.observation_links
responses = bundle.responses
univariate_misfits = {}
for resp in responses:
resp_values = list(resp.values)
univariate_misfits[resp.index] = {}
response_xaxis = list(bundle.indices)
for link in observation_links:
observation = link.observation
obs_values = list(observation.values)
obs_stds = list(observation.errors)
obs_xaxis = list(observation.x_axis)
misfits = []
for obs_index, obs_value in enumerate(obs_values):
obs_std = obs_stds[obs_index]
obs_x = obs_xaxis[obs_index]
if type(obs_x) == datetime:
obs_x = obs_x.isoformat()
resp_index = response_xaxis.index(obs_x)
misfits.append(
_calculate_misfit(
obs_value=obs_value,
obs_std=obs_std,
response_value=resp_values[resp_index],
x_value=obs_x,
))
univariate_misfits[resp.index][observation.name] = misfits
return_schema = {
"id":
id,
"name":
bundle.name,
"ensemble_id":
ensemble_id,
"realizations": [{
"name": resp.index,
"data": resp.values,
"summarized_misfits": {
misfit.observation_response_definition_link.observation.name:
misfit.value
for misfit in resp.misfits
},
"univariate_misfits": {
obs_name: misfits
for obs_name, misfits in univariate_misfits[
resp.index].items()
},
} for resp in responses],
"axis": {
"data": bundle.indices
},
}
if len(observation_links) > 0:
return_schema["observations"] = [
_obs_to_json(link.observation, link.active)
for link in observation_links
]
return return_schema
async def create_ensemble(*, db: Session = Db(), ens_in: js.EnsembleCreate):
update = None
if ens_in.update is not None:
update = ds.Update(
algorithm=ens_in.update.algorithm,
ensemble_reference_id=ens_in.update.ensemble_id,
)
ens = ds.Ensemble(
time_created=datetime.now(),
name=ens_in.name,
parent=update,
num_realizations=ens_in.realizations,
)
db.add(ens)
priors = {(p.group, p.key): ds.ParameterPrior(
group=p.group,
key=p.key,
function=p.function,
parameter_names=p.parameter_names,
parameter_values=p.parameter_values,
ensemble=[ens],
)
for p in ens_in.priors}
db.add_all(priors.values())
for index, param in enumerate(ens_in.parameters):
if len(param.values) != ens_in.realizations:
raise HTTPException(
status_code=422,
detail=
f"Length of Ensemble.parameters[{index}].values must be {ens_in.realizations}, got {len(param.values)}",
)
db.add_all(
ds.Parameter(
ensemble=ens,
name=p.name,
group=p.group,
prior=priors.get((p.group, p.name), None),
values=p.values,
) for p in ens_in.parameters)
existing_observations = {
obs.name: obs
for obs in db.query(ds.Observation).all()
}
# Adding only new observations. If observations exist, we link to the.
# existing ones. This will fail if users change the values in observation
# config file
observations = {}
to_be_created = []
for obs in ens_in.observations:
try:
observations[obs.name] = existing_observations[obs.name]
except KeyError:
new_obs = ds.Observation(
name=obs.name,
x_axis=obs.x_axis,
values=obs.values,
errors=obs.errors,
)
observations[obs.name] = new_obs
to_be_created.append(new_obs)
db.add_all(to_be_created)
response_definitions = {
response_name: ds.ResponseDefinition(name=response_name, ensemble=ens)
for response_name in ens_in.response_observation_link
}
db.add_all(response_definitions.values())
db.add_all(
ds.ObservationResponseDefinitionLink(
observation=observations[observation_name],
response_definition=response_definitions[response_name],
) for response_name, observation_name in
ens_in.response_observation_link.items())
db.commit()
return ens
async def read_response_by_id(*, db: Session = Db(), ensemble_id: int,
id: int):
return db.query(ds.Response).filter_by(id=id,
ensemble_id=ensemble_id).one()
async def parameters(*, db: Session = Db(), ensemble_id: int):
return (db.query(ds.Parameter).join(
ds.Parameter.prior,
isouter=True).filter(ds.Parameter.ensemble_id == ensemble_id).all())