def fast_sample( parameters: MutableMapping[str, Distribution], harmonics: Optional[int], sample_size: Optional[int], ) -> List[Dict[str, Record]]: if len(parameters) == 0: raise ValueError("Cannot study the sensitivity of no variables") if harmonics is None: harmonics = 4 if sample_size is None: sample_size = 1000 problem = _build_salib_problem(parameters) samples = fast_sampler.sample(problem, sample_size, M=harmonics) group_records = [] for dist in parameters.values(): records = [] for sample in samples: data = dict(zip(dist.index, sample[:dist.size])) record = NumericalRecord(data=data, index=dist.index) # type: ignore records.append(record) samples = np.delete(samples, list(range(dist.size)), axis=1) group_records.append(records) evaluations = [] for i in zip(*group_records): evaluation = dict(zip(parameters.keys(), i)) evaluations.append(evaluation) return evaluations
async def transform_output_sequence(self, mime: str, location: Path) -> Tuple[Record, ...]: if mime == "application/octet-stream": raise TypeError("Output record types must be NumericalRecord") raw_ensrecord = await get_serializer(mime).decode_from_path(location) return tuple( NumericalRecord(data=raw_record) for raw_record in raw_ensrecord)
async def _transmit(io_, transmitter: RecordTransmitter, data): record: Record = ( BlobRecord(data=data) if isinstance(data, bytes) else NumericalRecord(data=data) ) await transmitter.transmit_record(record) return (io_.get_name(), transmitter)
def _sync_eclsum_to_record(location: Path, smry_keys: List[str]) -> Dict[str, NumericalRecord]: eclsum = EclSum(str(location)) record_dict = {} for key in smry_keys: record_dict[key] = NumericalRecord(data=dict( zip(map(str, eclsum.dates), map(float, eclsum.numpy_vector(key))))) return record_dict
async def _load_record_from_file(file: Path, mime: str) -> Record: if mime == "application/octet-stream": async with aiofiles.open(str(file), mode="rb") as fb: contents_b: bytes = await fb.read() return BlobRecord(data=contents_b) else: serializer = get_serializer(mime) _record_data = await serializer.decode_from_path(file) return NumericalRecord(data=_record_data)
async def _transmit( io_: _IO, transmitter: "ert.data.RecordTransmitter", data: record_data ) -> Tuple[str, "ert.data.RecordTransmitter"]: record: Record = ( BlobRecord(data=data) if isinstance(data, bytes) else NumericalRecord(data=data) ) await transmitter.transmit_record(record) return (io_.name, transmitter)
async def test_simple_record_transmit( record_transmitter_factory_context: ContextManager[Callable[ [str], RecordTransmitter]], record_in, expected_data, storage_path, ): with record_transmitter_factory_context( storage_path=storage_path) as record_transmitter_factory: transmitter = record_transmitter_factory(name="some_name") await transmitter.transmit_record(record_in) assert transmitter.is_transmitted() with pytest.raises(RuntimeError, match="Record already transmitted"): await transmitter.transmit_record(NumericalRecord(data=[1, 2, 3]))
def record_factory(type: str): if type == "dir": dir_path = pathlib.Path(tmpdir) / "resources" / "test_dir" _files = [dir_path / "a.txt", dir_path / "b.txt"] with file_factory_context(tmpdir) as file_factory: file_factory(_files) tar_obj = io.BytesIO() with tarfile.open(fileobj=tar_obj, mode="w") as tar: tar.add(dir_path, arcname="") tardata = tar_obj.getvalue() return BlobRecord(data=tardata) elif type == "tree": return NumericalRecordTree( record_dict={ "a": NumericalRecord(data=[1, 2]), "b": NumericalRecord(data=[3, 4]), }) elif type == "blob": return BlobRecord(data=b"\xF0\x9F\xA6\x89") elif type == "eclsum": return None else: raise ValueError(f"type {type} not recognized")
def fast_sample( parameters: MutableMapping[str, Distribution], harmonics: Optional[int], sample_size: Optional[int], ) -> List[Dict[str, Record]]: """Construct the Numpy matrix with model inputs for the extended Fourier Amplitude Sensitivity test. The generated samples are intended to be used by fast_analyze() after a model has been run the returned data from this function. This function essentially wraps SALib.sample.fast_sampler.sample. Args: parameters: Collection of ert3 distributions harmonics: The inference parameter, in SALib called "M" sample_size: Number of samples to generate, called "N" in SALib """ if len(parameters) == 0: raise ValueError("Cannot study the sensitivity without any variables") if harmonics is None: harmonics = 4 if sample_size is None: sample_size = 1000 problem = _build_salib_problem(parameters) samples = fast_sampler.sample(problem, sample_size, M=harmonics) group_records = [] for dist in parameters.values(): # Loop over each parameter, let it be scalar or list-like records = [] for sample in samples: if dist.is_scalar: data = sample[0] else: data = dict(zip(dist.index, sample[:dist.size])) record = NumericalRecord(data=data, index=dist.index) records.append(record) samples = np.delete(samples, list(range(dist.size)), axis=1) group_records.append(records) evaluations = [] for i in zip(*group_records): evaluation = dict(zip(parameters.keys(), i)) evaluations.append(evaluation) return evaluations
def one_at_a_time(parameters: MutableMapping[str, Distribution], tail: Optional[float] = 0.99) -> List[Dict[str, Record]]: if len(parameters) == 0: raise ValueError("Cannot study the sensitivity of no variables") # Each variable will in turn # be explored as ppf(q) and ppf(1-q) as the two # extremal values. if tail is not None: q = (1 - tail) / 2 else: q = (1 - 0.99) / 2 evaluations = [] for parameter_name, dist in parameters.items(): lower = dist.ppf(q) upper = dist.ppf(1 - q) constant_records = set(parameters.keys()) constant_records.remove(parameter_name) if dist.is_scalar: for lim_val in (lower, upper): records = _build_base_records(constant_records, parameters) assert isinstance(lim_val, NumericalRecord) rec_value = NumericalRecord(data=lim_val.data) records[parameter_name] = rec_value evaluations.append(records) else: for idx in dist.index: for lim_val in (lower, upper): # Non-scalar parameters are treated as multiple parameters, # an explicit evaluation will be created for each variable records = _build_base_records(constant_records, parameters) rec_values = dist.ppf(0.5) # Modify only one of the base record values: rec_values.data[idx] = lim_val.data[idx] # type: ignore records[parameter_name] = rec_values evaluations.append(records) return evaluations
import cloudpickle import ert import pytest from ert.data import ( BlobRecord, BlobRecordTree, NumericalRecord, NumericalRecordTree, RecordTransmitter, RecordType, SerializationTransformation, ) from ert.data.record import _transmitter from ert_utils import tmp record123 = NumericalRecord(data=[1, 2, 3]) simple_records = pytest.mark.parametrize( ("record_in", "expected_data"), ( (record123, [1, 2, 3]), (NumericalRecord(data=1.0), 1.0), (NumericalRecord(data=1), 1.0), ( NumericalRecord(data=(1.0, 10.0, 42.0, 999.0)), [1.0, 10.0, 42.0, 999.0], ), ( NumericalRecord(data=[]), [], ),