def post_ensemble(*, db: Session = Depends(get_db), ens_in: js.EnsembleIn, experiment_id: UUID) -> ds.Ensemble: experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() active_reals = (ens_in.active_realizations if ens_in.active_realizations else list(range(ens_in.size))) if ens_in.size > 0: if max(active_reals) > ens_in.size - 1: raise exc.ExpectationError( f"Ensemble active realization index {max(active_reals)} out of realization range [0,{ ens_in.size - 1}]" ) if len(set(active_reals)) != len(active_reals): raise exc.ExpectationError( f"Non unique active realization index list not allowed {active_reals}" ) ens = ds.Ensemble( parameter_names=ens_in.parameter_names, response_names=ens_in.response_names, experiment=experiment, size=ens_in.size, userdata=ens_in.userdata, active_realizations=active_reals, ) db.add(ens) if ens_in.update_id: update_obj = db.query(ds.Update).filter_by(id=ens_in.update_id).one() update_obj.ensemble_result = ens db.commit() return ens
async def create_blob( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, ) -> None: """ Create a record which points to a blob on Azure Blob Storage. """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() file_obj = ds.File( filename="test", mimetype="mime/type", ) if HAS_AZURE_BLOB_STORAGE: key = f"{name}@{realization_index}@{uuid4()}" blob = azure_blob_container.get_blob_client(key) file_obj.az_container = (azure_blob_container.container_name, ) file_obj.az_blob = (key, ) else: pass db.add(file_obj) _create_record( db, ensemble, name, ds.RecordType.file, realization_index=realization_index, file=file_obj, )
async def add_block( *, db: Session = Depends(get_db), bh: BlobHandler = Depends(get_blob_handler), record: ds.Record = Depends(get_record_by_name), request: Request, block_index: int, ) -> None: """ Stage blocks to an existing azure blob record. """ db.add(await bh.stage_blob(record, request, block_index))
def _create_record( db: Session, record: ds.Record, ) -> ds.Record: nested = db.begin_nested() try: db.add(record) db.commit() except IntegrityError: # Assuming this is a UNIQUE constraint failure due to an existing # record_info with the same name and ensemble. Try to fetch the # record_info nested.rollback() record_info = record.record_info old_record_info = ( db.query(ds.RecordInfo) .filter_by(ensemble=record_info.ensemble, name=record_info.name) .one() ) # Check that the parameters match if record_info.record_class != old_record_info.record_class: raise exc.ConflictError( "Record class of new record does not match previous record class", new_record_class=record_info.record_class, old_record_class=old_record_info.record_class, ) if record_info.record_type != old_record_info.record_type: raise exc.ConflictError( "Record type of new record does not match previous record type", new_record_type=record_info.record_type, old_record_type=old_record_info.record_type, ) record = ds.Record( record_info=old_record_info, f64_matrix=record.f64_matrix, file=record.file, realization_index=record.realization_index, ) db.add(record) db.commit() return record
def create_update( *, db: Session = Depends(get_db), update: js.UpdateIn, ) -> js.UpdateOut: ensemble = db.query( ds.Ensemble).filter_by(id=update.ensemble_reference_id).one() update_obj = ds.Update( algorithm=update.algorithm, ensemble_reference_pk=ensemble.pk, ) db.add(update_obj) if update.observation_transformations: transformations = { t.name: t for t in update.observation_transformations } observation_ids = [t.observation_id for t in transformations.values()] observations = (db.query(ds.Observation).filter( ds.Observation.id.in_(observation_ids)).all()) observation_transformations = [ ds.ObservationTransformation( active_list=transformations[observation.name].active, scale_list=transformations[observation.name].scale, observation=observation, update=update_obj, ) for observation in observations ] db.add_all(observation_transformations) db.commit() return js.UpdateOut( id=update_obj.id, experiment_id=ensemble.experiment.id, algorithm=update_obj.algorithm, ensemble_reference_id=ensemble.id, )
def post_ensemble(*, db: Session = Depends(get_db), ens_in: js.EnsembleIn, experiment_id: UUID) -> ds.Ensemble: experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() ens = ds.Ensemble( parameter_names=ens_in.parameter_names, response_names=ens_in.response_names, experiment=experiment, size=ens_in.size, _metadata=ens_in.metadata, ) db.add(ens) if ens_in.update_id: update_obj = db.query(ds.Update).filter_by(id=ens_in.update_id).one() update_obj.ensemble_result = ens db.commit() return ens
def post_observation(*, db: Session = Depends(get_db), obs_in: js.ObservationIn, experiment_id: UUID) -> js.ObservationOut: experiment = db.query(ds.Experiment).filter_by(id=experiment_id).one() records = ([ db.query(ds.Record).filter_by(id=rec_id).one() for rec_id in obs_in.records ] if obs_in.records is not None else []) obs = ds.Observation( name=obs_in.name, x_axis=obs_in.x_axis, errors=obs_in.errors, values=obs_in.values, experiment=experiment, records=records, ) db.add(obs) db.commit() return _observation_from_db(obs)
async def post_ensemble_record_file( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, file: UploadFile = File(...), ) -> None: """ Assign an arbitrary file to the given `name` record. """ ensemble = _get_and_assert_ensemble(db, ensemble_id, name, realization_index) file_obj = ds.File( filename=file.filename, mimetype=file.content_type, ) if HAS_AZURE_BLOB_STORAGE: key = f"{name}@{realization_index}@{uuid4()}" blob = azure_blob_container.get_blob_client(key) await blob.upload_blob(file.file) file_obj.az_container = azure_blob_container.container_name file_obj.az_blob = key else: file_obj.content = await file.read() db.add(file_obj) _create_record( db, ensemble, name, ds.RecordType.file, realization_index=realization_index, file=file_obj, )
def post_experiments( *, db: Session = Depends(get_db), ens_in: js.ExperimentIn, ) -> js.ExperimentOut: experiment = ds.Experiment(name=ens_in.name) if ens_in.priors: db.add_all( ds.Prior( function=ds.PriorFunction.__members__[prior.function], experiment=experiment, name=name, argument_names=[ x[0] for x in prior if isinstance(x[1], (float, int)) ], argument_values=[ x[1] for x in prior if isinstance(x[1], (float, int)) ], ) for name, prior in ens_in.priors.items()) db.add(experiment) db.commit() return _experiment_from_db(experiment)
async def add_block( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, request: Request, block_index: int, ) -> None: """ Stage blocks to an existing azure blob record. """ ensemble = db.query(ds.Ensemble).filter_by(id=ensemble_id).one() block_id = str(uuid4()) file_block_obj = ds.FileBlock( ensemble=ensemble, block_id=block_id, block_index=block_index, record_name=name, realization_index=realization_index, ) record_obj = (db.query( ds.Record).filter_by(realization_index=realization_index).join( ds.RecordInfo).filter_by(ensemble_pk=ensemble.pk, name=name).one()) if HAS_AZURE_BLOB_STORAGE: key = record_obj.file.az_blob blob = azure_blob_container.get_blob_client(key) await blob.stage_block(block_id, await request.body()) else: file_block_obj.content = await request.body() db.add(file_block_obj) db.commit()
def _create_record( db: Session, ensemble: ds.Ensemble, name: str, record_type: ds.RecordType, record_class: ds.RecordClass = ds.RecordClass.other, prior: Optional[ds.Prior] = None, **kwargs: Any, ) -> ds.Record: record_info = ds.RecordInfo( ensemble=ensemble, name=name, record_class=record_class, record_type=record_type, prior=prior, ) record = ds.Record(record_info=record_info, **kwargs) nested = db.begin_nested() try: db.add(record) db.commit() except IntegrityError: # Assuming this is a UNIQUE constraint failure due to an existing # record_info with the same name and ensemble. Try to fetch the # record_info nested.rollback() old_record_info = (db.query(ds.RecordInfo).filter_by(ensemble=ensemble, name=name).one()) # Check that the parameters match if record_info.record_class != old_record_info.record_class: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ "error": "Record class of new record does not match previous record class", "new_record_class": str(record_info.record_class), "old_record_class": str(old_record_info.record_class), "name": name, "ensemble_id": str(ensemble.id), }, ) if record_info.record_type != old_record_info.record_type: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail={ "error": "Record type of new record does not match previous record type", "new_record_type": str(record_info.record_type), "old_record_type": str(old_record_info.record_type), "name": name, "ensemble_id": str(ensemble.id), }, ) record = ds.Record(record_info=old_record_info, **kwargs) db.add(record) db.commit() return record
async def post_ensemble_record_matrix( *, db: Session = Depends(get_db), ensemble_id: UUID, name: str, realization_index: Optional[int] = None, content_type: str = Header("application/json"), prior_id: Optional[UUID] = None, request: Request, ) -> js.RecordOut: """ Assign an n-dimensional float matrix, encoded in JSON, to the given `name` record. """ if content_type == "application/x-dataframe": logger.warning( "Content-Type with 'application/x-dataframe' is deprecated. Use 'text/csv' instead." ) content_type = "text/csv" ensemble = _get_and_assert_ensemble(db, ensemble_id, name, realization_index) is_parameter = name in ensemble.parameter_names is_response = name in ensemble.response_names if prior_id is not None and not is_parameter: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={ "error": "Priors can only be specified for parameter records", "name": name, "ensemble_id": str(ensemble_id), "realization_index": realization_index, "prior_id": str(prior_id), }, ) labels = None prior = ((db.query(ds.Prior).filter_by( id=prior_id, experiment_pk=ensemble.experiment_pk).one()) if prior_id else None) try: if content_type == "application/json": content = np.array(await request.json(), dtype=np.float64) elif content_type == "application/x-numpy": from numpy.lib.format import read_array stream = io.BytesIO(await request.body()) content = read_array(stream) elif content_type == "text/csv": stream = io.BytesIO(await request.body()) df = pd.read_csv(stream, index_col=0, float_precision="round_trip") content = df.values labels = [ [str(v) for v in df.columns.values], [str(v) for v in df.index.values], ] else: raise ValueError() except ValueError: if realization_index is None: message = f"Ensemble-wide record '{name}' for ensemble '{ensemble_id}' needs to be a matrix" else: message = f"Forward-model record '{name}' for ensemble '{ensemble_id}', realization {realization_index} needs to be a matrix" raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={ "error": message, "name": name, "ensemble_id": str(ensemble_id), "realization_index": realization_index, }, ) # Require that the dimensionality of an ensemble-wide parameter matrix is at least 2 if realization_index is None and is_parameter: if content.ndim <= 1: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail={ "error": "Ensemble-wide parameter record '{name}' for ensemble '{ensemble_id}'" "must have dimensionality of at least 2", "name": name, "ensemble_id": str(ensemble_id), "realization_index": realization_index, }, ) matrix_obj = ds.F64Matrix(content=content.tolist(), labels=labels) db.add(matrix_obj) record_class = ds.RecordClass.other if is_parameter: record_class = ds.RecordClass.parameter if is_response: record_class = ds.RecordClass.response return _create_record( db, ensemble, name, ds.RecordType.f64_matrix, record_class, prior, f64_matrix=matrix_obj, realization_index=realization_index, )