def test_generate_full_mc(tmpdir):
"""Test generation of full monte carlo"""
inputfile = TESTDATA / "config/design_input_mc_with_correls.xlsx"
input_dict = excel2dict_design(inputfile)
design = DesignMatrix()
design.generate(input_dict)
# Checking dimensions of design matrix
assert design.designvalues.shape == (500, 16)
# Checking reproducibility from distribution_seed
assert design.designvalues["PARAM1"].sum() == 17.419
# Write to disk and check some validity
tmpdir.chdir()
design.to_xlsx("designmatrix.xlsx")
assert Path("designmatrix.xlsx").exists
diskdesign = pd.read_excel("designmatrix.xlsx",
sheet_name="DesignSheet01",
engine="openpyxl")
assert "REAL" in diskdesign
assert "SENSNAME" in diskdesign
assert "SENSCASE" in diskdesign
assert not diskdesign.empty
diskdefaults = pd.read_excel("designmatrix.xlsx",
sheet_name="DefaultValues",
engine="openpyxl")
assert not diskdefaults.empty
assert len(diskdefaults.columns) == 2
def test_generate_onebyone(tmpdir):
"""Test generation of onebyone design"""
inputfile = TESTDATA / "config/design_input_example1.xlsx"
input_dict = excel2dict_design(inputfile)
design = DesignMatrix()
design.generate(input_dict)
# Checking dimensions of design matrix
assert design.designvalues.shape == (80, 10)
# Write to disk and check some validity
tmpdir.chdir()
design.to_xlsx("designmatrix.xlsx")
assert Path("designmatrix.xlsx").exists
diskdesign = pd.read_excel("designmatrix.xlsx", engine="openpyxl")
assert "REAL" in diskdesign
assert "SENSNAME" in diskdesign
assert "SENSCASE" in diskdesign
assert not diskdesign.empty
diskdefaults = pd.read_excel("designmatrix.xlsx",
sheet_name="DefaultValues",
engine="openpyxl")
assert not diskdefaults.empty
assert len(diskdefaults.columns) == 2
def test_generate_full_mc(tmpdir):
"""Test generation of full monte carlo"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
inputfile = (testdir + "/data/sensitivities/config/" +
"design_input_mc_with_correls.xlsx")
input_dict = excel2dict_design(inputfile)
design = DesignMatrix()
design.generate(input_dict)
# Checking dimensions of design matrix
assert design.designvalues.shape == (500, 16)
# Checking reproducibility from distribution_seed
assert design.designvalues["PARAM1"].sum() == 17.419
# Write to disk and check some validity
tmpdir.chdir()
design.to_xlsx("designmatrix.xlsx")
assert os.path.exists("designmatrix.xlsx")
diskdesign = pd.read_excel("designmatrix.xlsx", sheet_name="DesignSheet01")
assert "REAL" in diskdesign
assert "SENSNAME" in diskdesign
assert "SENSCASE" in diskdesign
assert not diskdesign.empty
diskdefaults = pd.read_excel("designmatrix.xlsx",
sheet_name="DefaultValues")
assert not diskdefaults.empty
assert len(diskdefaults.columns) == 2
def test_generate_onebyone(tmpdir):
"""Test generation of onebyone design"""
if "__file__" in globals():
# Easen up copying test code into interactive sessions
testdir = os.path.dirname(os.path.abspath(__file__))
else:
testdir = os.path.abspath(".")
inputfile = testdir + "/data/sensitivities/config/" + "design_input_example1.xlsx"
input_dict = excel2dict_design(inputfile)
design = DesignMatrix()
design.generate(input_dict)
# Checking dimensions of design matrix
assert design.designvalues.shape == (80, 10)
# Write to disk and check some validity
tmpdir.chdir()
design.to_xlsx("designmatrix.xlsx")
assert os.path.exists("designmatrix.xlsx")
diskdesign = pd.read_excel("designmatrix.xlsx")
assert "REAL" in diskdesign
assert "SENSNAME" in diskdesign
assert "SENSCASE" in diskdesign
assert not diskdesign.empty
diskdefaults = pd.read_excel("designmatrix.xlsx",
sheet_name="DefaultValues")
assert not diskdefaults.empty
assert len(diskdefaults.columns) == 2
def main():
"""fmudesign is a command line utility for generating design matrices
Wrapper for the the fmu.tools.sensitivities module"""
parser = get_parser()
args = parser.parse_args()
# Defaulted options should be reset to None, so that the other
# defaulting level inside _excel2dict can do its work.
if args.designinput == parser.get_default("designinput"):
args.designinput = None
if args.defaultvalues == parser.get_default("defaultvalues"):
args.defaultvalues = None
if args.general_input == parser.get_default("general_input"):
args.general_input = None
sheetnames = dict()
if args.designinput:
sheetnames["designinput"] = args.designinput
if args.defaultvalues:
sheetnames["defaultvalues"] = args.defaultvalues
if args.general_input:
sheetnames["general_input"] = args.general_input
if sheetnames:
print("Worksheets changed from default:")
print(sheetnames)
if isinstance(args.config, str):
if not Path(args.config).is_file():
raise IOError("Input file {} does not exist".format(args.config))
input_dict = excel2dict_design(args.config, sheetnames)
if args.config == args.destination:
raise IOError('Identical name "{}" have been provided for the input'
"file and the output file. "
" Exiting.....".format(args.config))
design = DesignMatrix()
design.generate(input_dict)
Path(args.destination).parent.mkdir(exist_ok=True, parents=True)
design.to_xlsx(args.destination)
def test_designmatrix():
"""Test the DesignMatrix class"""
design = DesignMatrix()
mock_dict = dict(
designtype="onebyone",
seeds="default",
repeats=10,
defaultvalues=dict(),
sensitivities=dict(rms_seed=dict(
seedname="RMS_SEED", senstype="seed", parameters=None)),
)
design.generate(mock_dict)
valid_designmatrix(design.designvalues)
assert len(design.designvalues) == 10
assert isinstance(design.defaultvalues, dict)
def main(args=None):
"""fmudesign is a command line utility for generating design matrices
Wrapper for the the fmu.tools.sensitivities module"""
args = _do_parse_args(args)
sheetnames = dict()
if args.designinput:
sheetnames["designinput"] = args.designinput
if args.defaultvalues:
sheetnames["defaultvalues"] = args.defaultvalues
if args.general_input:
sheetnames["general_input"] = args.general_input
if sheetnames:
print("Worksheets changed from default:")
print(sheetnames)
if isinstance(args.config, str):
if not os.path.isfile(args.config):
raise IOError("Input file {} does not exist".format(args.config))
input_dict = excel2dict_design(args.config, sheetnames)
if args.config == args.destination:
raise IOError('Identical name "{}" have been provided for the input'
"file and the output file. "
" Exiting.....".format(args.config))
design = DesignMatrix()
design.generate(input_dict)
folder = os.path.dirname(args.destination)
if folder and not os.path.exists(folder):
os.makedirs(folder)
design.to_xlsx(args.destination)
# input_dict=yaml.load(input_file)
path = "./tests/data/sensitivities/config/"
prefix = "design_input_"
postfix = ".xlsx"
config = [
"example1",
"example2",
"example_velocities",
"singlereference",
"singlereference_and_seed",
"default_no_seed",
"onebyone",
"background_no_seed",
"background_extseeds",
"mc_with_correls",
"montecarlo_full",
"mc_corr_depend",
]
for input in range(len(config)):
filename = path + prefix + config[input] + postfix
print("Reading {}".format(filename))
input_dict = excel2dict_design(filename)
# input_dict.to_yaml(input_dict, 'examples/output/'+config[input]+'.yaml')
design = DesignMatrix()
design.generate(input_dict)
design.to_xlsx("examples/output/design_" + config[input] + postfix)
# if design.backgroundvalues is not None:
# design.background_to_excel('examples/output/background.xlsx')
def test_prediction_rejection_sampled_ensemble(tmpdir):
"""Test making a design matrix for prediction realizations based on an
ensemble made with manual history matching (rejection sampling).
In the use-case this test is modelled on, the design matrix is used
to set up a prediction ensemble where each DATA file points to another
Eclipse run on disk which contains the history, identified by the
realization index ("HMREAL") in the history match run.
"""
tmpdir.chdir()
general_input = pd.DataFrame(data=[
["designtype", "onebyone"],
["repeats", 3], # This matches the number of HM-samples we have.
["rms_seeds", None], # Geogrid from HM realization is used
["background", "hmrealizations.xlsx"],
["distribution_seed", None],
])
defaultvalues = pd.DataFrame(
columns=["param_name", "default_value"],
data=[
# All background parameters must be mentioned in
# DefaultValues (but these defaults are not used in
# this particular test scenario)
["HMREAL", "-1"],
["ORAT", 6000],
["RESTARTPATH", "FOO"],
["HMITER", "-1"],
],
)
# Background to separate file, these define some history realizations that
# all scenarios should run over:
pd.DataFrame(
columns=["RESTARTPATH", "HMREAL", "HMITER"],
data=[
["/scratch/foo/2020a_hm3/", 31, 3],
["/scratch/foo/2020a_hm3/", 38, 3],
["/scratch/foo/2020a_hm3/", 54, 3],
],
).to_excel("hmrealizations.xlsx")
writer = pd.ExcelWriter("designinput.xlsx", engine="openpyxl")
general_input.to_excel(writer,
sheet_name="general_input",
index=False,
header=None)
pd.DataFrame(
columns=[
"sensname",
"numreal",
"type",
"param_name",
"dist_name",
"dist_param1",
"dist_param2",
],
data=[
["ref", None, "background", None],
["oil_rate", None, "dist", "ORAT", "uniform", 5000, 9000],
],
).to_excel(writer, sheet_name="design_input", index=False)
defaultvalues.to_excel(writer, sheet_name="defaultvalues", index=False)
writer.save()
dict_design = excel2dict_design("designinput.xlsx")
design = DesignMatrix()
design.generate(dict_design)
assert set(
design.designvalues["RESTARTPATH"]) == {"/scratch/foo/2020a_hm3/"}
assert set(design.designvalues["HMITER"]) == {3}
assert all(design.designvalues["REAL"] == [0, 1, 2, 3, 4, 5])
assert all(design.designvalues["SENSNAME"] == [
"ref",
"ref",
"ref",
"oil_rate",
"oil_rate",
"oil_rate",
])
# This is the most important bit in this test function, that the realization
# list in the background xlsx is repeated for each sensitivity:
assert all(design.designvalues["HMREAL"] == [31, 38, 54, 31, 38, 54])