def fast_sample(
parameters: MutableMapping[str, Distribution],
harmonics: Optional[int],
sample_size: Optional[int],
) -> List[Dict[str, Record]]:
if len(parameters) == 0:
raise ValueError("Cannot study the sensitivity of no variables")
if harmonics is None:
harmonics = 4
if sample_size is None:
sample_size = 1000
problem = _build_salib_problem(parameters)
samples = fast_sampler.sample(problem, sample_size, M=harmonics)
group_records = []
for dist in parameters.values():
records = []
for sample in samples:
data = dict(zip(dist.index, sample[:dist.size]))
record = NumericalRecord(data=data,
index=dist.index) # type: ignore
records.append(record)
samples = np.delete(samples, list(range(dist.size)), axis=1)
group_records.append(records)
evaluations = []
for i in zip(*group_records):
evaluation = dict(zip(parameters.keys(), i))
evaluations.append(evaluation)
return evaluations
async def transform_output_sequence(self, mime: str,
location: Path) -> Tuple[Record, ...]:
if mime == "application/octet-stream":
raise TypeError("Output record types must be NumericalRecord")
raw_ensrecord = await get_serializer(mime).decode_from_path(location)
return tuple(
NumericalRecord(data=raw_record) for raw_record in raw_ensrecord)
async def _transmit(io_, transmitter: RecordTransmitter, data):
record: Record = (
BlobRecord(data=data)
if isinstance(data, bytes)
else NumericalRecord(data=data)
)
await transmitter.transmit_record(record)
return (io_.get_name(), transmitter)
def _sync_eclsum_to_record(location: Path,
smry_keys: List[str]) -> Dict[str, NumericalRecord]:
eclsum = EclSum(str(location))
record_dict = {}
for key in smry_keys:
record_dict[key] = NumericalRecord(data=dict(
zip(map(str, eclsum.dates), map(float, eclsum.numpy_vector(key)))))
return record_dict
async def _load_record_from_file(file: Path, mime: str) -> Record:
if mime == "application/octet-stream":
async with aiofiles.open(str(file), mode="rb") as fb:
contents_b: bytes = await fb.read()
return BlobRecord(data=contents_b)
else:
serializer = get_serializer(mime)
_record_data = await serializer.decode_from_path(file)
return NumericalRecord(data=_record_data)
async def _transmit(
io_: _IO, transmitter: "ert.data.RecordTransmitter", data: record_data
) -> Tuple[str, "ert.data.RecordTransmitter"]:
record: Record = (
BlobRecord(data=data)
if isinstance(data, bytes)
else NumericalRecord(data=data)
)
await transmitter.transmit_record(record)
return (io_.name, transmitter)
async def test_simple_record_transmit(
record_transmitter_factory_context: ContextManager[Callable[
[str], RecordTransmitter]],
record_in,
expected_data,
storage_path,
):
with record_transmitter_factory_context(
storage_path=storage_path) as record_transmitter_factory:
transmitter = record_transmitter_factory(name="some_name")
await transmitter.transmit_record(record_in)
assert transmitter.is_transmitted()
with pytest.raises(RuntimeError, match="Record already transmitted"):
await transmitter.transmit_record(NumericalRecord(data=[1, 2, 3]))
def record_factory(type: str):
if type == "dir":
dir_path = pathlib.Path(tmpdir) / "resources" / "test_dir"
_files = [dir_path / "a.txt", dir_path / "b.txt"]
with file_factory_context(tmpdir) as file_factory:
file_factory(_files)
tar_obj = io.BytesIO()
with tarfile.open(fileobj=tar_obj, mode="w") as tar:
tar.add(dir_path, arcname="")
tardata = tar_obj.getvalue()
return BlobRecord(data=tardata)
elif type == "tree":
return NumericalRecordTree(
record_dict={
"a": NumericalRecord(data=[1, 2]),
"b": NumericalRecord(data=[3, 4]),
})
elif type == "blob":
return BlobRecord(data=b"\xF0\x9F\xA6\x89")
elif type == "eclsum":
return None
else:
raise ValueError(f"type {type} not recognized")
def fast_sample(
parameters: MutableMapping[str, Distribution],
harmonics: Optional[int],
sample_size: Optional[int],
) -> List[Dict[str, Record]]:
"""Construct the Numpy matrix with model inputs for the extended Fourier
Amplitude Sensitivity test. The generated samples are intended to be used by
fast_analyze() after a model has been run the returned data from this
function.
This function essentially wraps SALib.sample.fast_sampler.sample.
Args:
parameters: Collection of ert3 distributions
harmonics: The inference parameter, in SALib called "M"
sample_size: Number of samples to generate, called "N" in SALib
"""
if len(parameters) == 0:
raise ValueError("Cannot study the sensitivity without any variables")
if harmonics is None:
harmonics = 4
if sample_size is None:
sample_size = 1000
problem = _build_salib_problem(parameters)
samples = fast_sampler.sample(problem, sample_size, M=harmonics)
group_records = []
for dist in parameters.values():
# Loop over each parameter, let it be scalar or list-like
records = []
for sample in samples:
if dist.is_scalar:
data = sample[0]
else:
data = dict(zip(dist.index, sample[:dist.size]))
record = NumericalRecord(data=data, index=dist.index)
records.append(record)
samples = np.delete(samples, list(range(dist.size)), axis=1)
group_records.append(records)
evaluations = []
for i in zip(*group_records):
evaluation = dict(zip(parameters.keys(), i))
evaluations.append(evaluation)
return evaluations
def one_at_a_time(parameters: MutableMapping[str, Distribution],
tail: Optional[float] = 0.99) -> List[Dict[str, Record]]:
if len(parameters) == 0:
raise ValueError("Cannot study the sensitivity of no variables")
# Each variable will in turn # be explored as ppf(q) and ppf(1-q) as the two
# extremal values.
if tail is not None:
q = (1 - tail) / 2
else:
q = (1 - 0.99) / 2
evaluations = []
for parameter_name, dist in parameters.items():
lower = dist.ppf(q)
upper = dist.ppf(1 - q)
constant_records = set(parameters.keys())
constant_records.remove(parameter_name)
if dist.is_scalar:
for lim_val in (lower, upper):
records = _build_base_records(constant_records, parameters)
assert isinstance(lim_val, NumericalRecord)
rec_value = NumericalRecord(data=lim_val.data)
records[parameter_name] = rec_value
evaluations.append(records)
else:
for idx in dist.index:
for lim_val in (lower, upper):
# Non-scalar parameters are treated as multiple parameters,
# an explicit evaluation will be created for each variable
records = _build_base_records(constant_records, parameters)
rec_values = dist.ppf(0.5)
# Modify only one of the base record values:
rec_values.data[idx] = lim_val.data[idx] # type: ignore
records[parameter_name] = rec_values
evaluations.append(records)
return evaluations
import cloudpickle
import ert
import pytest
from ert.data import (
BlobRecord,
BlobRecordTree,
NumericalRecord,
NumericalRecordTree,
RecordTransmitter,
RecordType,
SerializationTransformation,
)
from ert.data.record import _transmitter
from ert_utils import tmp
record123 = NumericalRecord(data=[1, 2, 3])
simple_records = pytest.mark.parametrize(
("record_in", "expected_data"),
(
(record123, [1, 2, 3]),
(NumericalRecord(data=1.0), 1.0),
(NumericalRecord(data=1), 1.0),
(
NumericalRecord(data=(1.0, 10.0, 42.0, 999.0)),
[1.0, 10.0, 42.0, 999.0],
),
(
NumericalRecord(data=[]),
[],
),
