async def finalize_blob( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, ) -> None: """ Commit all staged blocks to a blob record """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() record_obj = (db.query( ds.Record).filter_by(realization_index=realization_index).join( ds.RecordInfo).filter_by(ensemble_pk=ensemble.pk, name=name).one()) submitted_blocks = list( db.query(ds.FileBlock).filter_by( record_name=name, ensemble_pk=ensemble.pk, realization_index=realization_index, ).all()) if HAS_AZURE_BLOB_STORAGE: key = record_obj.file.az_blob blob = azure_blob_container.get_blob_client(key) block_ids = [ block.block_id for block in sorted(submitted_blocks, key=lambda x: x.block_index) ] await blob.commit_block_list(block_ids) else: data = b"".join([block.content for block in submitted_blocks]) record_obj.file.content = data
async def get_ensemble_response_dataframe(*, db: Session = Depends(get_db), ensemble_id: UUID, response_name: str) -> Response: ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() records = (db.query(ds.Record).filter( ds.Record.realization_index != None).join(ds.RecordInfo).filter_by( ensemble_pk=ensemble.pk, name=response_name, record_class=ds.RecordClass.response, )).all() df_list = [] for record in records: data_df = pd.DataFrame(record.f64_matrix.content) labels = record.f64_matrix.labels if labels is not None: # if the realization is more than 1D array # the next line will produce ValueError exception data_df.index = [record.realization_index] data_df.columns = labels[0] df_list.append(data_df) return Response( content=pd.concat(df_list, axis=0).to_csv().encode(), media_type="text/csv", )
def post_ensemble(*, db: Session = Depends(get_db), ens_in: js.EnsembleIn, experiment_id: UUID) -> ds.Ensemble: experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() active_reals = (ens_in.active_realizations if ens_in.active_realizations else list(range(ens_in.size))) if ens_in.size > 0: if max(active_reals) > ens_in.size - 1: raise exc.ExpectationError( f"Ensemble active realization index {max(active_reals)} out of realization range [0,{ ens_in.size - 1}]" ) if len(set(active_reals)) != len(active_reals): raise exc.ExpectationError( f"Non unique active realization index list not allowed {active_reals}" ) ens = ds.Ensemble( parameter_names=ens_in.parameter_names, response_names=ens_in.response_names, experiment=experiment, size=ens_in.size, userdata=ens_in.userdata, active_realizations=active_reals, ) db.add(ens) if ens_in.update_id: update_obj = db.query(ds.Update).filter_by(id=ens_in.update_id).one() update_obj.ensemble_result = ens db.commit() return ens
def get_record_by_name( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, ) -> ds.Record: try: return ( db.query(ds.Record) .filter_by(realization_index=realization_index) .join(ds.RecordInfo) .filter_by(name=name) .join(ds.Ensemble) .filter_by(id=ensemble_id) .one() ) except NoResultFound as e: pass if realization_index is not None: return ( db.query(ds.Record) .filter_by(realization_index=None) .join(ds.RecordInfo) .filter_by( name=name, record_type=ds.RecordType.f64_matrix, ) .join(ds.Ensemble) .filter_by(id=ensemble_id) .one() ) raise exc.NotFoundError(f"Record not found")
def get_update( *, db: Session = Depends(get_db), update_id: UUID, ) -> js.UpdateOut: update_obj = db.query(ds.Update).filter_by(id=update_id).one() return _update_from_db(update_obj)
async def create_blob( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, ) -> None: """ Create a record which points to a blob on Azure Blob Storage. """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() file_obj = ds.File( filename="test", mimetype="mime/type", ) if HAS_AZURE_BLOB_STORAGE: key = f"{name}@{realization_index}@{uuid4()}" blob = azure_blob_container.get_blob_client(key) file_obj.az_container = (azure_blob_container.container_name, ) file_obj.az_blob = (key, ) else: pass db.add(file_obj) _create_record( db, ensemble, name, ds.RecordType.file, realization_index=realization_index, file=file_obj, )
async def post_record_observations( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, observation_ids: List[UUID] = Body(...), ) -> None: if realization_index is None: record_obj = _get_ensemble_record(db, ensemble_id, name) else: record_obj = _get_forward_model_record(db, ensemble_id, name, realization_index) observations = (db.query(ds.Observation).filter( ds.Observation.id.in_(observation_ids)).all()) if observations: record_obj.observations = observations flag_modified(record_obj, "observations") db.commit() else: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={ "error": f"Observations {observation_ids} not found!", "name": name, "ensemble_id": str(ensemble_id), }, )
async def get_record_labels( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, ) -> List[str]: """ Get the list of record data labels. If the record is not a group record the list of labels will contain only the name of the record Example - Group record: data - {"a": 4, "b":42, "c": 2} return: ["a", "b", "c"] - Ensemble record: data - [4, 42, 2, 32] return: [] """ record = ( db.query(ds.Record) .join(ds.RecordInfo) .filter_by(name=name) .join(ds.Ensemble) .filter_by(id=ensemble_id) .first() ) if record is None: raise exc.NotFoundError(f"Record not found") if record.f64_matrix and record.f64_matrix.labels: return record.f64_matrix.labels[0] return []
def get_ensemble_responses(*, db: Session = Depends(get_db), ensemble_id: UUID) -> Mapping[str, ds.Record]: return { rec.name: rec for rec in (db.query(ds.Record).join(ds.RecordInfo).filter_by( record_class=ds.RecordClass.response).join(ds.Ensemble).filter_by( id=ensemble_id).all()) }
async def get_observation_userdata( *, db: Session = Depends(get_db), obs_id: UUID, ) -> Mapping[str, Any]: """ Get userdata json """ obs = db.query(ds.Observation).filter_by(id=obs_id).one() return obs.userdata
def _get_forward_model_record(db: Session, ensemble_id: UUID, name: str, realization_index: int) -> ds.Record: try: ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() return (db.query(ds.Record).filter_by( realization_index=realization_index).join(ds.RecordInfo).filter_by( ensemble_pk=ensemble.pk, name=name, ).one()) except NoResultFound: raise HTTPException( status_code=404, detail={ "error": f"Forward-model record '{name}' for ensemble '{ensemble_id}', realization {realization_index} not found!", "name": name, "ensemble_id": str(ensemble_id), }, )
async def get_experiment_userdata( *, db: Session = Depends(get_db), experiment_id: UUID, ) -> Mapping[str, Any]: """ Get userdata json """ experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() return experiment.userdata
async def get_ensemble_userdata( *, db: Session = Depends(get_db), ensemble_id: UUID, ) -> Mapping[str, Any]: """ Get userdata json """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() return ensemble.userdata
async def get_response_misfits( *, db: Session = Depends(get_db), ensemble_id: UUID, response_name: str, realization_index: Optional[int] = None, summary_misfits: bool = False, ) -> Response: """ Compute univariate misfits for response(s) """ response_query = (db.query(ds.Record).filter( ds.Record.observations != None).join(ds.RecordInfo).filter_by( name=response_name, record_type=ds.RecordType.f64_matrix, ).join(ds.Ensemble).filter_by(id=ensemble_id)) if realization_index is not None: responses = [ response_query.filter( ds.Record.realization_index == realization_index).one() ] else: responses = response_query.all() observation_df = None response_dict = {} for response in responses: data_df = pd.DataFrame(response.f64_matrix.content) labels = response.f64_matrix.labels if labels is not None: data_df.columns = labels[0] data_df.index = labels[1] response_dict[response.realization_index] = data_df if observation_df is None: # currently we expect only a single observation object, while # later in the future this might change obs = response.observations[0] observation_df = pd.DataFrame(data={ "values": obs.values, "errors": obs.errors }, index=obs.x_axis) try: result_df = calculate_misfits_from_pandas(response_dict, observation_df, summary_misfits) except Exception as misfits_exc: raise exc.UnprocessableError( f"Unable to compute misfits: {misfits_exc}") return Response( content=result_df.to_csv().encode(), media_type="text/csv", )
def create_update( *, db: Session = Depends(get_db), update: js.UpdateIn, ) -> js.UpdateOut: ensemble = db.query( ds.Ensemble).filter_by(id=update.ensemble_reference_id).one() update_obj = ds.Update( algorithm=update.algorithm, ensemble_reference_pk=ensemble.pk, ) db.add(update_obj) if update.observation_transformations: transformations = { t.name: t for t in update.observation_transformations } observation_ids = [t.observation_id for t in transformations.values()] observations = (db.query(ds.Observation).filter( ds.Observation.id.in_(observation_ids)).all()) observation_transformations = [ ds.ObservationTransformation( active_list=transformations[observation.name].active, scale_list=transformations[observation.name].scale, observation=observation, update=update_obj, ) for observation in observations ] db.add_all(observation_transformations) db.commit() return js.UpdateOut( id=update_obj.id, experiment_id=ensemble.experiment.id, algorithm=update_obj.algorithm, ensemble_reference_id=ensemble.id, )
def get_records_by_name( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, ) -> List[ds.Record]: records = ( db.query(ds.Record) .filter_by(realization_index=realization_index) .join(ds.RecordInfo) .filter_by(name=name) .join(ds.Ensemble) .filter_by(id=ensemble_id) ).all() if not records: records = ( db.query(ds.Record) .join(ds.RecordInfo) .filter_by( name=name, record_type=ds.RecordType.f64_matrix, ) .join(ds.Ensemble) .filter_by(id=ensemble_id) ).all() if not records: records = ( db.query(ds.Record) .filter_by(realization_index=None) .join(ds.RecordInfo) .filter_by(name=name) .join(ds.Ensemble) .filter_by(id=ensemble_id) ).all() if not records: raise exc.NotFoundError(f"Record not found") return records
def new_record( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, ) -> ds.Record: """ Get ensemble and verify that no record with the name `name` exists """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() q = ( db.query(ds.Record) .join(ds.RecordInfo) .filter_by(ensemble_pk=ensemble.pk, name=name) ) if ( ensemble.size != -1 and realization_index is not None and realization_index not in ensemble.active_realizations ): raise exc.ExpectationError( f"Realization index {realization_index} outside of allowed realization indices {ensemble.active_realizations}" ) q = q.filter( (ds.Record.realization_index == None) | (ds.Record.realization_index == realization_index) ) if q.count() > 0: raise exc.ConflictError( f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' already exists", ) return ds.Record( record_info=ds.RecordInfo( ensemble=ensemble, name=name, ), realization_index=realization_index, )
def _get_and_assert_ensemble(db: Session, ensemble_id: UUID, name: str, realization_index: Optional[int]) -> ds.Ensemble: """ Get ensemble and verify that no record with the name `name` exists """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() q = (db.query(ds.Record).join(ds.RecordInfo).filter_by( ensemble_pk=ensemble.pk, name=name)) if realization_index is not None: if realization_index not in range( ensemble.size) and ensemble.size != -1: raise HTTPException( status_code=status.HTTP_417_EXPECTATION_FAILED, detail={ "error": f"Ensemble '{name}' ('{ensemble_id}') does have a 'size' " f"of {ensemble.size}. The posted record is targeting " f"'realization_index' {realization_index} which is out " f"of bounds.", "name": name, "ensemble_id": str(ensemble_id), }, ) q = q.filter((ds.Record.realization_index == None) | (ds.Record.realization_index == realization_index)) if q.count() > 0: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ "error": f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' already exists", "name": name, "ensemble_id": str(ensemble_id), }, ) return ensemble
async def replace_experiment_userdata( *, db: Session = Depends(get_db), experiment_id: UUID, body: Any = Body(...), ) -> None: """ Assign new userdata json """ experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() experiment.userdata = body db.commit()
def post_ensemble(*, db: Session = Depends(get_db), ens_in: js.EnsembleIn, experiment_id: UUID) -> ds.Ensemble: experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() ens = ds.Ensemble( parameter_names=ens_in.parameter_names, response_names=ens_in.response_names, experiment=experiment, size=ens_in.size, _metadata=ens_in.metadata, ) db.add(ens) if ens_in.update_id: update_obj = db.query(ds.Update).filter_by(id=ens_in.update_id).one() update_obj.ensemble_result = ens db.commit() return ens
async def replace_observation_userdata( *, db: Session = Depends(get_db), obs_id: UUID, body: Any = Body(...), ) -> None: """ Assign new userdata json """ obs = db.query(ds.Observation).filter_by(id=obs_id).one() obs.userdata = body db.commit()
async def replace_ensemble_userdata( *, db: Session = Depends(get_db), ensemble_id: UUID, body: Any = Body(...), ) -> None: """ Assign new userdata json """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() ensemble.userdata = body db.commit()
async def patch_experiment_userdata( *, db: Session = Depends(get_db), experiment_id: UUID, body: Any = Body(...), ) -> None: """ Update userdata json """ experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() experiment.userdata.update(body) flag_modified(experiment, "userdata") db.commit()
def post_observation(*, db: Session = Depends(get_db), obs_in: js.ObservationIn, experiment_id: UUID) -> js.ObservationOut: experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() records = ([ db.query(ds.Record).filter_by(id=rec_id).one() for rec_id in obs_in.records ] if obs_in.records is not None else []) obs = ds.Observation( name=obs_in.name, x_axis=obs_in.x_axis, errors=obs_in.errors, values=obs_in.values, experiment=experiment, records=records, ) db.add(obs) db.commit() return _observation_from_db(obs)
async def patch_observation_userdata( *, db: Session = Depends(get_db), obs_id: UUID, body: Any = Body(...), ) -> None: """ Update userdata json """ obs = db.query(ds.Observation).filter_by(id=obs_id).one() obs.userdata.update(body) flag_modified(obs, "userdata") db.commit()
async def patch_ensemble_userdata( *, db: Session = Depends(get_db), ensemble_id: UUID, body: Any = Body(...), ) -> None: """ Update userdata json """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() ensemble.userdata.update(body) flag_modified(ensemble, "userdata") db.commit()
async def get_record_data( *, db: Session = Depends(get_db), record_id: UUID, accept: Optional[str] = Header(default="application/json"), ) -> Any: if accept == "application/x-dataframe": logger.warning( "Accept with 'application/x-dataframe' is deprecated. Use 'text/csv' instead." ) accept = "text/csv" record = db.query(ds.Record).filter_by(id=record_id).one() return await _get_record_data(record, accept)
async def post_record_observations( *, db: Session = Depends(get_db), record: ds.Record = Depends(get_record_by_name), observation_ids: List[UUID] = Body(...), ) -> None: observations = ( db.query(ds.Observation).filter(ds.Observation.id.in_(observation_ids)).all() ) if observations: record.observations = observations db.commit() else: raise exc.UnprocessableError(f"Observations {observation_ids} not found!")
async def add_block( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, request: Request, block_index: int, ) -> None: """ Stage blocks to an existing azure blob record. """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() block_id = str(uuid4()) file_block_obj = ds.FileBlock( ensemble=ensemble, block_id=block_id, block_index=block_index, record_name=name, realization_index=realization_index, ) record_obj = (db.query( ds.Record).filter_by(realization_index=realization_index).join( ds.RecordInfo).filter_by(ensemble_pk=ensemble.pk, name=name).one()) if HAS_AZURE_BLOB_STORAGE: key = record_obj.file.az_blob blob = azure_blob_container.get_blob_client(key) await blob.stage_block(block_id, await request.body()) else: file_block_obj.content = await request.body() db.add(file_block_obj) db.commit()
def get_observations_with_transformation( *, db: Session = Depends(get_db), ensemble_id: UUID) -> List[js.ObservationOut]: ens = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() experiment = ens.experiment update = ens.parent transformations = ({ trans.observation.name: trans for trans in update.observation_transformations } if update is not None else {}) return [ _observation_from_db(obs, transformations) for obs in experiment.observations ]