def test_discrete_properties(self, data_grid):
"""Test extracting statsitics on discrete properties"""
data_grid["properties"] = {
"FACIES": {"name": "reek_sim_facies2.roff"},
}
data_grid["selectors"] = {
"ZONE": {"name": "reek_sim_zone.roff"},
}
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert set(qcp.dataframe.columns) == set(
[
"Avg_Weighted",
"Avg",
"Count",
"FACIES",
"PROPERTY",
"ZONE",
"SOURCE",
"ID",
]
)
assert qcp._proptypes_all[0] == "DISC"
assert list(qcp.dataframe["PROPERTY"].unique()) == ["FACIES"]
assert set(qcp.dataframe["FACIES"].unique()) == set(
["FINESAND", "COARSESAND", "SHALE"]
)
row = qcp.dataframe[
(qcp.dataframe["ZONE"] == "Total") & (qcp.dataframe["FACIES"] == "FINESAND")
]
assert row["Avg"].values == pytest.approx(0.4024, abs=0.001)
def test_read_eclipse_init(self, data_grid):
"""Test reading property from INIT-file"""
data_grid["grid"] = "REEK.EGRID"
data_grid["properties"] = {
"PORO": {
"name": "PORO",
"pfile": "REEK.INIT"
},
"PERM": {
"name": "PERMX",
"pfile": "REEK.INIT"
},
}
data_grid["selectors"] = {
"REGION": {
"name": "FIPNUM",
"pfile": "REEK.INIT"
},
}
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert ["REEK"] == list(qcp.dataframe["ID"].unique())
assert qcp.dataframe[(qcp.dataframe["PROPERTY"] == "PORO") & (
qcp.dataframe["REGION"] == "2")]["Avg"].values == pytest.approx(
0.1661, abs=0.001)
def test_codenames():
data = data_orig.copy()
qcp = QCProperties()
stat_no_code = qcp.get_grid_statistics(data)
data["selectors"] = {
"ZONE": {
"name": "reek_sim_zone.roff",
"codes": {
1: "TOP",
2: "MID"
}
},
"FACIES": {
"name": "reek_sim_facies2.roff",
},
}
stat = qcp.get_grid_statistics(data, reuse=True)
assert set(["TOP", "MID", "Below_Low_reek", "Total"]) == {
x
for x in list(stat.dataframe["ZONE"].unique()) if x is not None
}
assert set(["Below_Top_reek", "Below_Mid_reek", "Below_Low_reek",
"Total"]) == {
x
for x in list(stat_no_code.dataframe["ZONE"].unique())
if x is not None
}
def test_read_eclipse():
data = data_orig.copy()
data["grid"] = "REEK.EGRID"
data["properties"] = {
"PORO": {
"name": "PORO",
"pfile": "REEK.INIT"
},
"PERM": {
"name": "PERMX",
"pfile": "REEK.INIT"
},
}
data["selectors"] = {
"REGION": {
"name": "FIPNUM",
"pfile": "REEK.INIT"
},
}
qcp = QCProperties()
qcp.get_grid_statistics(data)
assert set(["REEK"]) == set(qcp.dataframe["ID"].unique())
assert qcp.dataframe[(qcp.dataframe["PROPERTY"] == "PORO") & (
qcp.dataframe["REGION"] == "2")]["Avg"].values == pytest.approx(
0.1661, abs=0.001)
def test_no_selector_combos(self, data_grid):
"""Test running without selector_combos"""
data_grid["selector_combos"] = False
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert ["Total"] == list(
qcp.dataframe[qcp.dataframe["ZONE"] == "Total"]["FACIES"].unique())
def test_propstatset():
qcp = QCProperties()
qcp.get_grid_statistics(GRIDDATA)
qcp.get_well_statistics(WELLDATA)
qcp.get_bwell_statistics(BWELLDATA)
assert len(qcp.dataframe["ID"].unique()) == 3
def test_set_id(self, data_grid):
"""Test extracting statsitics on continous properties"""
data_grid["name"] = "Test_case"
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert ["Test_case"] == list(qcp.dataframe["ID"].unique())
qcp.get_grid_statistics(data_grid)
assert ["Test_case", "Test_case(1)"] == qcp.dataframe["ID"].unique().tolist()
def test_no_selectors(self, data_grid):
"""Test running without selectors"""
data_grid.pop("selectors", None)
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert len(qcp.dataframe) == 2
assert qcp.dataframe[qcp.dataframe["PROPERTY"] == "PORO"][
"Avg"
].values == pytest.approx(0.1677, abs=0.001)
def test_auto_combination(self, data_grid, data_wells, data_bwells):
"""Tests combining statistic """
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert len(qcp.dataframe["ID"].unique()) == 1
qcp.get_well_statistics(data_wells)
assert len(qcp.dataframe["ID"].unique()) == 2
qcp.get_bwell_statistics(data_bwells)
assert len(qcp.dataframe["ID"].unique()) == 3
def test_gridprops(self, data_grid):
"""Test extracting statsitics from grid properties"""
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert set(qcp.dataframe["PROPERTY"].unique()) == set(["PORO", "PERM"])
row = qcp.dataframe[(qcp.dataframe["ZONE"] == "Total")
& (qcp.dataframe["FACIES"] == "Total")
& (qcp.dataframe["PROPERTY"] == "PORO")]
assert row["Avg"].values == pytest.approx(0.1677, abs=0.001)
assert row["Max"].values == pytest.approx(0.3613, abs=0.001)
def test_propstatset_auto_combination():
qcp = QCProperties()
qcp.get_grid_statistics(GRIDDATA)
assert len(qcp.dataframe["ID"].unique()) == 1
qcp.get_well_statistics(WELLDATA)
assert len(qcp.dataframe["ID"].unique()) == 2
qcp.get_bwell_statistics(BWELLDATA, reuse=True)
assert len(qcp.dataframe["ID"].unique()) == 3
def test_statistics():
data = data_orig.copy()
data["name"] = "Test_case"
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert set(stat.dataframe.columns) == set([
"Avg_Weighted",
"Avg",
"FACIES",
"Max",
"Min",
"P10",
"P90",
"PROPERTY",
"Stddev",
"ZONE",
"SOURCE",
"ID",
])
assert list(stat.dataframe["ID"].unique())[0] == data["name"]
assert set(stat.dataframe["PROPERTY"].unique()) == set(["PORO", "PERM"])
assert stat.dataframe[stat.dataframe["PROPERTY"] ==
"PORO"]["Avg"].max() == pytest.approx(0.3138,
abs=0.001)
row = stat.dataframe[(stat.dataframe["ZONE"] == "Total")
& (stat.dataframe["FACIES"] == "Total")
& (stat.dataframe["PROPERTY"] == "PORO")]
assert row["Avg"].values == pytest.approx(0.1677, abs=0.001)
def test_no_selectors():
data = data_orig.copy()
data.pop("selectors", None)
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert set(stat.property_dataframe.columns) == set(["PORO", "PERM"])
def test_extract_statistics_update_filter_parameter():
"""Test changing filters after initialization"""
data = data_orig.copy()
data["selectors"] = ["reek_sim_zone.roff"]
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert stat.property_dataframe["PORO"].mean() == pytest.approx(0.1677,
abs=0.001)
assert set(stat.property_dataframe.columns) == set([
"PORO",
"PERM",
"reek_sim_zone.roff",
])
stat.extract_statistics(filters={
"reek_sim_facies2.roff": {
"include": ["FINESAND", "COARSESAND"],
}
}, )
assert set(stat.property_dataframe.columns) == set([
"PORO",
"PERM",
"reek_sim_facies2.roff",
"reek_sim_zone.roff",
])
assert ["FINESAND", "COARSESAND"] == list(
stat.property_dataframe["reek_sim_facies2.roff"].unique())
assert stat.property_dataframe["PORO"].mean() == pytest.approx(0.2374,
abs=0.001)
def test_statistics_no_combos():
data = data_orig.copy()
data["selector_combos"] = False
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert ["Total"] == list(
stat.dataframe[stat.dataframe["ZONE"] == "Total"]["FACIES"].unique())
def test_full_dataframe():
data = data_orig.copy()
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert stat.property_dataframe["PORO"].mean() == pytest.approx(0.1677,
abs=0.001)
assert stat.property_dataframe["PORO"].max() == pytest.approx(0.3613,
abs=0.001)
assert set(stat.property_dataframe.columns) == set(
["PORO", "PERM", "ZONE", "FACIES"])
def test_continous_properties(self, data_grid):
"""Test extracting statsitics on continous properties"""
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert set(qcp.dataframe.columns) == set([
"Avg_Weighted",
"Avg",
"Count",
"FACIES",
"Max",
"Min",
"P10",
"P90",
"PROPERTY",
"Stddev",
"ZONE",
"SOURCE",
"ID",
])
assert qcp._proptypes_all[0] == "CONT"
def check_gridstatistics(self, project, data):
"""
Extract statistics per action and check if property value is
within user specified limits.
Returns a dataframe with results
"""
qcp = QCProperties()
results = []
QCC.print_info("Checking status for items in actions...")
for action in self.ldata.actions:
# extract parameters from actions
data_upd = self._extract_parameters_from_action(data, action)
selectors, calculation = self._get_selecors_and_calculation(action)
# Create datframe with statistics
dframe = qcp.get_grid_statistics(project=project, data=data_upd)
# Get value from statistics for given property and selectors
value = self._get_statistical_value(dframe, action["property"],
calculation, selectors)
status = "OK"
if "warn_outside" in action:
if not action["warn_outside"][0] <= value <= action[
"warn_outside"][1]:
status = "WARN"
if not action["stop_outside"][0] <= value <= action[
"stop_outside"][1]:
status = "STOP"
result = collections.OrderedDict()
result["PROPERTY"] = action["property"]
result["SELECTORS"] = f"{list(selectors.values())}"
result["FILTERS"] = "yes" if "filters" in action else "no"
result["CALCULATION"] = calculation
result["VALUE"] = value
result["STOP_LIMITS"] = f"{action['stop_outside']}"
result["WARN_LIMITS"] = (f"{action['warn_outside']}"
if "warn_outside" in action else "-")
result["STATUS"] = status
result["DESCRIPTION"] = action.get("description", "-")
results.append(result)
dfr = self.make_report(results,
reportfile=self.ldata.reportfile,
nametag=self.ldata.nametag)
return dfr
def test_multiple_filters():
data = data_orig.copy()
data.pop("selectors", None)
data["multiple_filters"] = {
"test1": {
"reek_sim_facies2.roff": {
"include": ["SHALE"],
}
},
"test2": {
"reek_sim_facies2.roff": {
"exclude": ["SHALE"],
}
},
}
qcp = QCProperties()
qcp.get_grid_statistics(data)
assert set(["test1", "test2"]) == set(qcp.dataframe["ID"].unique())
assert qcp.dataframe[(qcp.dataframe["PROPERTY"] == "PORO") & (
qcp.dataframe["ID"] == "test1")]["Avg"].values == pytest.approx(
0.1183, abs=0.001)
def test_multiple_filters(self, data_grid):
"""Test running two statistics extractions using multiple_filters"""
data_grid.pop("selectors", None)
data_grid["multiple_filters"] = {
"test1": {
"reek_sim_facies2.roff": {
"include": ["SHALE"],
}
},
"test2": {
"reek_sim_facies2.roff": {
"exclude": ["SHALE"],
}
},
}
qcp = QCProperties()
qcp.get_grid_statistics(data_grid)
assert set(["test1", "test2"]) == set(qcp.dataframe["ID"].unique())
assert qcp.dataframe[
(qcp.dataframe["PROPERTY"] == "PORO") & (qcp.dataframe["ID"] == "test1")
]["Avg"].values == pytest.approx(0.1183, abs=0.001)
def test_get_value():
data = data_orig.copy()
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert stat.get_value("PORO") == pytest.approx(0.1677, abs=0.001)
assert stat.get_value("PORO",
calculation="Max") == pytest.approx(0.3613,
abs=0.001)
conditions = {"ZONE": "Below_Top_reek", "FACIES": "COARSESAND"}
assert stat.get_value("PORO",
conditions=conditions) == pytest.approx(0.3117,
abs=0.001)
conditions = {"ZONE": "Below_Top_reek"}
assert stat.get_value("PORO",
conditions=conditions) == pytest.approx(0.1595,
abs=0.001)
def test_extract_statistics_update_filter_values():
"""Test changing filters after initialization"""
data = data_orig.copy()
data["selectors"] = {
"ZONE": {
"name": "reek_sim_zone.roff",
"exclude": ["Below_Top_reek"]
},
"FACIES": {
"name": "reek_sim_facies2.roff",
"include": ["FINESAND", "COARSESAND"],
},
}
data["filters"] = {
"reek_sim_facies2.roff": {
"include": ["FINESAND", "COARSESAND"],
}
}
qcp = QCProperties()
stat = qcp.get_grid_statistics(data)
assert "Below_Top_reek" not in list(
stat.property_dataframe["ZONE"].unique())
assert ["FINESAND",
"COARSESAND"] == list(stat.property_dataframe["FACIES"].unique())
assert stat.property_dataframe["PORO"].mean() == pytest.approx(0.2390,
abs=0.001)
stat.extract_statistics(
filters={"reek_sim_facies2.roff": {
"include": ["SHALE"],
}})
assert "Below_Top_reek" not in list(
stat.property_dataframe["ZONE"].unique())
assert ["SHALE"] == list(stat.property_dataframe["FACIES"].unique())
assert stat.property_dataframe["PORO"].mean() == pytest.approx(0.1155,
abs=0.001)
def run(
self,
data: dict,
project: Union[object, str] = None,
):
"""Main routine for evaulating if statistics from 3D grids is
within user specified thresholds.
The routine depends on existing fmu.tools functionality for
extracting property statistics from 3D grids.
Args:
data (dict or str): The input data either as a Python dictionary or
a path to a YAML file
project (Union[object, str]): For usage inside RMS
"""
self._data = self.handle_data(data, project)
# TO-DO:
# self._validate_input(self._data)
data = self._data
QCC.verbosity = data.get("verbosity", 0)
# parse data that are special for this check
QCC.print_info("Parsing additional data...")
self.ldata = _LocalData()
self.ldata.parse_data(data)
qcp = QCProperties()
results = []
QCC.print_info("Checking status for items in actions...")
for action in self.ldata.actions:
# extract parameters from actions and compute statistics
data_upd = self._extract_parameters_from_action(data, action)
stat = qcp.get_grid_statistics(
project=project, data=data_upd, reuse=True, qcdata=self.gdata
)
selectors = (
list(action.get("selectors").values())
if "selectors" in action
else None
)
# Extract mean value if calculation is not given
if "calculation" in action:
calculation = action["calculation"]
else:
calculation = "Avg" if action.get("codename") is None else "Percent"
# Get value from statistics for given property and selectors
value = stat.get_value(
action["property"],
conditions=action.get("selectors"),
calculation=calculation,
codename=action.get("codename"),
)
status = "OK"
if "warn_outside" in action:
if not action["warn_outside"][0] <= value <= action["warn_outside"][1]:
status = "WARN"
if not action["stop_outside"][0] <= value <= action["stop_outside"][1]:
status = "STOP"
result = collections.OrderedDict()
result["PROPERTY"] = action["property"]
result["SELECTORS"] = f"{selectors}"
result["FILTERS"] = "yes" if "filters" in action else "no"
result["CALCULATION"] = calculation
result["VALUE"] = value
result["STOP_LIMITS"] = f"{action['stop_outside']}"
result["WARN_LIMITS"] = (
f"{action['warn_outside']}" if "warn_outside" in action else "-"
)
result["STATUS"] = status
result["DESCRIPTION"] = action.get("description", "-")
results.append(result)
dfr = self.make_report(
results, reportfile=self.ldata.reportfile, nametag=self.ldata.nametag
)
if len(dfr[dfr["STATUS"] == "WARN"]) > 0:
print(dfr[dfr["STATUS"] == "WARN"])
if len(dfr[dfr["STATUS"] == "STOP"]) > 0:
print(dfr[dfr["STATUS"] == "STOP"], file=sys.stderr)
msg = "One or more actions has status = STOP"
QCC.force_stop(msg)
print(
"\n== QC forward check {} ({}) finished ==".format(
self.__class__.__name__, self.ldata.nametag
)
)
