def __init__(
self,
app,
ensembles: Optional[list] = None,
csvfile_parameters: pathlib.Path = None,
csvfile_smry: pathlib.Path = None,
time_index: str = "monthly",
column_keys: Optional[list] = None,
drop_constants: bool = True,
):
super().__init__()
WEBVIZ_ASSETS.add(
pathlib.Path(webviz_subsurface.__file__).parent / "_assets" /
"css" / "container.css")
self.theme = app.webviz_settings["theme"]
self.time_index = time_index
self.column_keys = column_keys
self.ensembles = ensembles
self.csvfile_parameters = csvfile_parameters
self.csvfile_smry = csvfile_smry
if ensembles is not None:
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.pmodel = ParametersModel(
dataframe=self.emodel.load_parameters(),
theme=self.theme,
drop_constants=drop_constants,
)
self.vmodel = SimulationTimeSeriesModel(
dataframe=self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys),
theme=self.theme,
)
elif self.csvfile_parameters is None:
raise ValueError(
"Either ensembles or csvfile_parameters must be specified")
else:
self.pmodel = ParametersModel(
dataframe=read_csv(csvfile_parameters),
theme=self.theme,
drop_constants=drop_constants,
)
if self.csvfile_smry is not None:
self.vmodel = SimulationTimeSeriesModel(
dataframe=read_csv(csvfile_smry),
theme=self.theme.plotly_theme)
else:
self.vmodel = None
self.set_callbacks(app)
def __init__(
self,
app: dash.Dash,
ensembles: Optional[list] = None,
statistics_file: str = "share/results/tables/gridpropstatistics.csv",
csvfile_statistics: pathlib.Path = None,
csvfile_smry: pathlib.Path = None,
surface_renaming: Optional[dict] = None,
time_index: str = "monthly",
column_keys: Optional[list] = None,
):
super().__init__()
WEBVIZ_ASSETS.add(
pathlib.Path(webviz_subsurface.__file__).parent / "_assets" /
"css" / "container.css")
# TODO(Sigurd) fix this once we get a separate webviz_settings parameter
self.theme: WebvizConfigTheme = app.webviz_settings["theme"]
self.time_index = time_index
self.column_keys = column_keys
self.statistics_file = statistics_file
self.ensembles = ensembles
self.csvfile_statistics = csvfile_statistics
self.csvfile_smry = csvfile_smry
self.surface_folders: Union[dict, None]
if ensembles is not None:
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.pmodel = PropertyStatisticsModel(
dataframe=self.emodel.load_csv(
csv_file=pathlib.Path(self.statistics_file)),
theme=self.theme,
)
self.vmodel = SimulationTimeSeriesModel(
dataframe=self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys),
theme=self.theme,
)
self.surface_folders = {
ens: folder / "share" / "results" / "maps" / ens
for ens, folder in self.emodel.ens_folders.items()
}
else:
self.pmodel = PropertyStatisticsModel(
dataframe=read_csv(csvfile_statistics), theme=self.theme)
self.vmodel = SimulationTimeSeriesModel(
dataframe=read_csv(csvfile_smry), theme=self.theme)
self.surface_folders = None
self.surface_renaming = surface_renaming if surface_renaming else {}
self.set_callbacks(app)
def __init__(
self,
app: dash.Dash,
ensembles: list,
wells: Optional[List[str]] = None,
):
super().__init__()
if wells is None:
self.column_keys = ["WBHP:*"]
else:
self.column_keys = [f"WBHP:{well}" for well in wells]
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.smry = self.emodel.load_smry(time_index="raw",
column_keys=self.column_keys)
self.theme = app.webviz_settings["theme"]
self.set_callbacks(app)
class ReservoirSimulationTimeSeriesOneByOne(WebvizPluginABC):
"""Visualizes reservoir simulation time series data for sensitivity studies based \
on a design matrix.
A tornado plot can be calculated interactively for each date/vector by selecting a date.
After selecting a date individual sensitivities can be selected to highlight the realizations
run with that sensitivity.
---
**Two main options for input data: Aggregated and read from UNSMRY.**
**Using aggregated data**
* **`csvfile_smry`:** Aggregated `csv` file for volumes with `REAL`, `ENSEMBLE`, `DATE` and \
vector columns (absolute path or relative to config file).
* **`csvfile_parameters`:** Aggregated `csv` file for sensitivity information with `REAL`, \
`ENSEMBLE`, `SENSNAME` and `SENSCASE` columns (absolute path or relative to config file).
**Using simulation time series data directly from `UNSMRY` files**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
* **`column_keys`:** List of vectors to extract. If not given, all vectors \
from the simulations will be extracted. Wild card asterisk `*` can be used.
* **`sampling`:** Time separation between extracted values. Can be e.g. `monthly` (default) or \
`yearly`.
**Common optional settings for both input options**
* **`initial_vector`:** Initial vector to display
* **`line_shape_fallback`:** Fallback interpolation method between points. Vectors identified as \
rates or phase ratios are always backfilled, vectors identified as cumulative (totals) are \
always linearly interpolated. The rest use the fallback.
Supported options:
* `linear` (default)
* `backfilled`
* `hv`, `vh`, `hvh`, `vhv` and `spline` (regular Plotly options).
---
!> It is **strongly recommended** to keep the data frequency to a regular frequency (like \
`monthly` or `yearly`). This applies to both csv input and when reading from `UNSMRY` \
(controlled by the `sampling` key). This is because the statistics and fancharts are calculated \
per DATE over all realizations in an ensemble, and the available dates should therefore not \
differ between individual realizations of an ensemble.
**Using aggregated data**
* [Example of csvfile_smry]\
(https://github.com/equinor/webviz-subsurface-testdata/blob/master/aggregated_data/smry.csv).
* [Example of csvfile_parameters]\
(https://github.com/equinor/webviz-subsurface-testdata/blob/master/aggregated_data/parameters.csv).
**Using simulation time series data directly from `.UNSMRY` files**
Time series data are extracted automatically from the `UNSMRY` files in the individual
realizations, using the `fmu-ensemble` library. The `SENSNAME` and `SENSCASE` values are read
directly from the `parameters.txt` files of the individual realizations, assuming that these
exist. If the `SENSCASE` of a realization is `p10_p90`, the sensitivity case is regarded as a
**Monte Carlo** style sensitivity, otherwise the case is evaluated as a **scalar** sensitivity.
?> Using the `UNSMRY` method will also extract metadata like units, and whether the vector is a \
rate, a cumulative, or historical. Units are e.g. added to the plot titles, while rates and \
cumulatives are used to decide the line shapes in the plot. Aggregated data may on the other \
speed up the build of the app, as processing of `UNSMRY` files can be slow for large models.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
"""
ENSEMBLE_COLUMNS = [
"REAL",
"ENSEMBLE",
"DATE",
"SENSCASE",
"SENSNAME",
"SENSTYPE",
"RUNPATH",
]
TABLE_STAT: List[Tuple[str, dict]] = [
("Sensitivity", {}),
("Case", {}),
] + table_statistics_base()
# pylint: disable=too-many-arguments
def __init__(
self,
app: dash.Dash,
csvfile_smry: Path = None,
csvfile_parameters: Path = None,
ensembles: list = None,
column_keys: list = None,
initial_vector: str = None,
sampling: str = "monthly",
line_shape_fallback: str = "linear",
) -> None:
super().__init__()
self.time_index = sampling
self.column_keys = column_keys
self.csvfile_smry = csvfile_smry
self.csvfile_parameters = csvfile_parameters
if csvfile_smry and ensembles:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile_smry" and "csvfile_parameters" or '
'"ensembles"')
if csvfile_smry and csvfile_parameters:
smry = read_csv(csvfile_smry)
parameters = read_csv(csvfile_parameters)
parameters["SENSTYPE"] = parameters.apply(
lambda row: find_sens_type(row.SENSCASE), axis=1)
self.smry_meta = None
elif ensembles:
self.ens_paths = {
ensemble: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ensemble]
for ensemble in ensembles
}
self.emodel = EnsembleSetModel(ensemble_paths=self.ens_paths)
smry = self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys)
self.smry_meta = self.emodel.load_smry_meta(
column_keys=self.column_keys, )
# Extract realizations and sensitivity information
parameters = get_realizations(ensemble_paths=self.ens_paths,
ensemble_set_name="EnsembleSet")
else:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile_smry" and "csvfile_parameters" or '
'"ensembles"')
self.data = pd.merge(smry, parameters, on=["ENSEMBLE", "REAL"])
self.smry_cols = [
c for c in self.data.columns
if c not in ReservoirSimulationTimeSeriesOneByOne.ENSEMBLE_COLUMNS
and not historical_vector(c, self.smry_meta,
False) in self.data.columns
]
self.initial_vector = (initial_vector if initial_vector
and initial_vector in self.smry_cols else
self.smry_cols[0])
self.line_shape_fallback = set_simulation_line_shape_fallback(
line_shape_fallback)
self.tornadoplot = TornadoPlot(app, parameters, allow_click=True)
self.uid = uuid4()
self.theme = app.webviz_settings["theme"]
self.set_callbacks(app)
def ids(self, element: str) -> str:
"""Generate unique id for dom element"""
return f"{element}-id-{self.uid}"
@property
def tour_steps(self) -> List[dict]:
return [
{
"id":
self.ids("layout"),
"content":
("Dashboard displaying time series from a sensitivity study."),
},
{
"id":
self.ids("graph-wrapper"),
"content":
("Selected time series displayed per realization. "
"Click in the plot to calculate tornadoplot for the "
"corresponding date, then click on the tornado plot to "
"highlight the corresponding sensitivity."),
},
{
"id":
self.ids("table"),
"content":
("Table statistics for all sensitivities for the selected date."
),
},
*self.tornadoplot.tour_steps,
{
"id": self.ids("vector"),
"content": "Select time series"
},
{
"id": self.ids("ensemble"),
"content": "Select ensemble"
},
]
@property
def ensemble_selector(self) -> html.Div:
"""Dropdown to select ensemble"""
return html.Div(
style={"paddingBottom": "30px"},
children=html.Label(children=[
html.Span("Ensemble:", style={"font-weight": "bold"}),
dcc.Dropdown(
id=self.ids("ensemble"),
options=[{
"label": i,
"value": i
} for i in list(self.data["ENSEMBLE"].unique())],
clearable=False,
value=list(self.data["ENSEMBLE"].unique())[0],
persistence=True,
persistence_type="session",
),
]),
)
@property
def smry_selector(self) -> html.Div:
"""Dropdown to select ensemble"""
return html.Div(
style={"paddingBottom": "30px"},
children=html.Label(children=[
html.Span("Time series:", style={"font-weight": "bold"}),
dcc.Dropdown(
id=self.ids("vector"),
options=[{
"label":
f"{simulation_vector_description(vec)} ({vec})",
"value": vec,
} for vec in self.smry_cols],
clearable=False,
value=self.initial_vector,
persistence=True,
persistence_type="session",
),
]),
)
@property
def initial_date(self) -> datetime.date:
df = self.data[self.data["ENSEMBLE"] == self.data["ENSEMBLE"].unique()
[0]]
return df["DATE"].max()
def add_webvizstore(self) -> List[Tuple[Callable, list]]:
return ([(
read_csv,
[
{
"csv_file": self.csvfile_smry
},
{
"csv_file": self.csvfile_parameters
},
],
)] if self.csvfile_smry and self.csvfile_parameters else
self.emodel.webvizstore + [(
get_realizations,
[{
"ensemble_paths": self.ens_paths,
"ensemble_set_name": "EnsembleSet",
}],
)])
@property
def layout(self) -> html.Div:
return html.Div(children=[
wcc.FlexBox(
id=self.ids("layout"),
children=[
html.Div(
style={"flex": 2},
children=[
wcc.FlexBox(children=[
self.ensemble_selector,
self.smry_selector,
dcc.Store(
id=self.ids("date-store"),
storage_type="session",
),
], ),
wcc.FlexBox(children=[
html.Div(
id=self.ids("graph-wrapper"),
style={"height": "450px"},
children=wcc.Graph(
id=self.ids("graph"),
clickData={
"points": [{
"x": self.initial_date
}]
},
),
),
]),
html.Div(children=[
html.Div(
id=self.ids("table_title"),
style={"textAlign": "center"},
children="",
),
DataTable(
id=self.ids("table"),
sort_action="native",
filter_action="native",
page_action="native",
page_size=10,
),
], ),
],
),
html.Div(
style={"flex": 1},
id=self.ids("tornado-wrapper"),
children=self.tornadoplot.layout,
),
],
),
])
# pylint: disable=too-many-statements
def set_callbacks(self, app: dash.Dash) -> None:
@app.callback(
[
# Output(self.ids("date-store"), "children"),
Output(self.ids("table"), "data"),
Output(self.ids("table"), "columns"),
Output(self.ids("table_title"), "children"),
Output(self.tornadoplot.storage_id, "data"),
],
[
Input(self.ids("ensemble"), "value"),
Input(self.ids("graph"), "clickData"),
Input(self.ids("vector"), "value"),
],
)
def _render_date(ensemble: str, clickdata: dict,
vector: str) -> Tuple[list, list, str, str]:
"""Store selected date and tornado input. Write statistics
to table"""
try:
date = clickdata["points"][0]["x"]
except TypeError as exc:
raise PreventUpdate from exc
data = filter_ensemble(self.data, ensemble, [vector])
data = data.loc[data["DATE"].astype(str) == date]
table_rows, table_columns = calculate_table(data, vector)
return (
# json.dumps(f"{date}"),
table_rows,
table_columns,
(f"{simulation_vector_description(vector)} ({vector})" +
("" if get_unit(self.smry_meta, vector) is None else
f" [{get_unit(self.smry_meta, vector)}]")),
json.dumps({
"ENSEMBLE":
ensemble,
"data":
data[["REAL", vector]].values.tolist(),
"number_format":
"#.4g",
"unit": ("" if get_unit(self.smry_meta, vector) is None
else get_unit(self.smry_meta, vector)),
}),
)
@app.callback(
Output(self.ids("graph"), "figure"),
[
Input(self.tornadoplot.click_id, "data"),
Input(self.tornadoplot.high_low_storage_id, "data"),
],
[
State(self.ids("ensemble"), "value"),
State(self.ids("vector"), "value"),
State(self.ids("graph"), "clickData"),
State(self.ids("graph"), "figure"),
],
) # pylint: disable=too-many-branches, too-many-locals
def _render_tornado(
tornado_click_data_str: Union[str, None],
high_low_storage: dict,
ensemble: str,
vector: str,
date_click: dict,
figure: dict,
) -> dict:
"""Update graph with line coloring, vertical line and title"""
if dash.callback_context.triggered is None:
raise PreventUpdate
ctx = dash.callback_context.triggered[0]["prop_id"].split(".")[0]
tornado_click: Union[dict,
None] = (json.loads(tornado_click_data_str)
if tornado_click_data_str else None)
if tornado_click:
reset_click = tornado_click["sens_name"] is None
else:
reset_click = False
# Draw initial figure and redraw if ensemble/vector changes
if ctx in ["", self.tornadoplot.high_low_storage_id
] or reset_click:
if historical_vector(vector, self.smry_meta,
True) in self.data.columns:
data = filter_ensemble(
self.data,
ensemble,
[
vector,
historical_vector(vector, self.smry_meta, True)
],
)
else:
data = filter_ensemble(self.data, ensemble, [vector])
line_shape = get_simulation_line_shape(
line_shape_fallback=self.line_shape_fallback,
vector=vector,
smry_meta=self.smry_meta,
)
traces = [{
"type": "line",
"marker": {
"color": "grey"
},
"hoverinfo": "x+y+text",
"hovertext": f"Real: {r}",
"x": df["DATE"],
"y": df[vector],
"customdata": r,
"line": {
"shape": line_shape
},
"meta": {
"SENSCASE": df["SENSCASE"].values[0],
"SENSTYPE": df["SENSTYPE"].values[0],
},
"name": ensemble,
"legendgroup": ensemble,
"showlegend": r == data["REAL"][0],
} for r, df in data.groupby(["REAL"])]
if historical_vector(vector, self.smry_meta,
True) in data.columns:
hist = data[data["REAL"] == data["REAL"][0]]
traces.append({
"type":
"line",
"x":
hist["DATE"],
"y":
hist[historical_vector(vector, self.smry_meta, True)],
"line": {
"shape": line_shape,
"color": "black",
"width": 3,
},
"name":
"History",
"legendgroup":
"History",
"showlegend":
True,
})
# traces[0]["hoverinfo"] = "x"
figure = {
"data": traces,
"layout": {
"margin": {
"t": 60
},
"hovermode": "closest"
},
}
# Update line colors if a sensitivity is selected in tornado
# pylint: disable=too-many-nested-blocks
if tornado_click and tornado_click["sens_name"] in high_low_storage:
if ctx == self.tornadoplot.high_low_storage_id:
tornado_click["real_low"] = high_low_storage[
tornado_click["sens_name"]].get("real_low")
tornado_click["real_high"] = high_low_storage[
tornado_click["sens_name"]].get("real_high")
if reset_click:
add_legend = True
for trace in figure["data"]:
if trace["name"] != "History":
if add_legend:
trace["showlegend"] = True
add_legend = False
else:
trace["showlegend"] = False
trace["marker"] = {"color": "grey"}
trace["opacity"] = 1
trace["name"] = ensemble
trace["legendgroup"] = ensemble
trace["hoverinfo"] = "all"
trace["hovertext"] = f"Real: {trace['customdata']}"
else:
add_legend_low = True
add_legend_high = True
for trace in figure["data"]:
if trace["name"] != "History":
if trace["customdata"] in tornado_click[
"real_low"]:
trace["marker"] = {
"color":
self.theme.plotly_theme["layout"]
["colorway"][0]
}
trace["opacity"] = 1
trace["legendgroup"] = "real_low"
trace["hoverinfo"] = "all"
trace["name"] = (
"Below ref" if trace["meta"]["SENSTYPE"]
== "mc" else trace["meta"]["SENSCASE"])
if add_legend_low:
add_legend_low = False
trace["showlegend"] = True
else:
trace["showlegend"] = False
elif trace["customdata"] in tornado_click[
"real_high"]:
trace["marker"] = {
"color":
self.theme.plotly_theme["layout"]
["colorway"][1]
}
trace["opacity"] = 1
trace["legendgroup"] = "real_high"
trace["hoverinfo"] = "all"
trace["name"] = (
"Above ref" if trace["meta"]["SENSTYPE"]
== "mc" else trace["meta"]["SENSCASE"])
if add_legend_high:
add_legend_high = False
trace["showlegend"] = True
else:
trace["showlegend"] = False
else:
trace["marker"] = {"color": "lightgrey"}
trace["opacity"] = 0.02
trace["showlegend"] = False
trace["hoverinfo"] = "skip"
date = date_click["points"][0]["x"]
if figure is None:
raise PreventUpdate
ymin = min([min(trace["y"]) for trace in figure["data"]])
ymax = max([max(trace["y"]) for trace in figure["data"]])
figure["layout"]["shapes"] = [{
"type": "line",
"x0": date,
"x1": date,
"y0": ymin,
"y1": ymax
}]
figure["layout"]["title"] = (
f"Date: {date}, "
f"Sensitivity: {tornado_click['sens_name'] if tornado_click else None}"
)
figure["layout"]["yaxis"] = {
"title":
f"{simulation_vector_description(vector)} ({vector})" +
("" if get_unit(self.smry_meta, vector) is None else
f" [{get_unit(self.smry_meta, vector)}]")
}
figure["layout"]["legend"] = {
"orientation": "h",
# "traceorder": "reversed",
"y": 1.1,
"x": 1,
"xanchor": "right",
}
figure["layout"] = self.theme.create_themed_layout(
figure["layout"])
return figure
def __init__(
self,
app: dash.Dash,
csvfile_smry: Path = None,
csvfile_parameters: Path = None,
ensembles: list = None,
column_keys: list = None,
initial_vector: str = None,
sampling: str = "monthly",
line_shape_fallback: str = "linear",
) -> None:
super().__init__()
self.time_index = sampling
self.column_keys = column_keys
self.csvfile_smry = csvfile_smry
self.csvfile_parameters = csvfile_parameters
if csvfile_smry and ensembles:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile_smry" and "csvfile_parameters" or '
'"ensembles"')
if csvfile_smry and csvfile_parameters:
smry = read_csv(csvfile_smry)
parameters = read_csv(csvfile_parameters)
parameters["SENSTYPE"] = parameters.apply(
lambda row: find_sens_type(row.SENSCASE), axis=1)
self.smry_meta = None
elif ensembles:
self.ens_paths = {
ensemble: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ensemble]
for ensemble in ensembles
}
self.emodel = EnsembleSetModel(ensemble_paths=self.ens_paths)
smry = self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys)
self.smry_meta = self.emodel.load_smry_meta(
column_keys=self.column_keys, )
# Extract realizations and sensitivity information
parameters = get_realizations(ensemble_paths=self.ens_paths,
ensemble_set_name="EnsembleSet")
else:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile_smry" and "csvfile_parameters" or '
'"ensembles"')
self.data = pd.merge(smry, parameters, on=["ENSEMBLE", "REAL"])
self.smry_cols = [
c for c in self.data.columns
if c not in ReservoirSimulationTimeSeriesOneByOne.ENSEMBLE_COLUMNS
and not historical_vector(c, self.smry_meta,
False) in self.data.columns
]
self.initial_vector = (initial_vector if initial_vector
and initial_vector in self.smry_cols else
self.smry_cols[0])
self.line_shape_fallback = set_simulation_line_shape_fallback(
line_shape_fallback)
self.tornadoplot = TornadoPlot(app, parameters, allow_click=True)
self.uid = uuid4()
self.theme = app.webviz_settings["theme"]
self.set_callbacks(app)
class ParameterAnalysis(WebvizPluginABC):
"""This plugin visualizes parameter distributions and statistics. /
for FMU ensembles, and can be used to investigate parameter correlations /
on reservoir simulation time series data.
---
**Input data can be provided in two ways: Aggregated or read from ensembles stored on scratch.**
**Using aggregated data**
* **`csvfile_parameters`:** Aggregated `csv` file with `REAL`, `ENSEMBLE` and parameter columns. \
(absolute path or relative to config file).
* **`csvfile_smry`:** (Optional) Aggregated `csv` file for volumes with `REAL`, `ENSEMBLE`, `DATE` \
and vector columns (absolute path or relative to config file).
**Using raw ensemble data stored in realization folders**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
* **`column_keys`:** List of vectors to extract. If not given, all vectors \
from the simulations will be extracted. Wild card asterisk `*` can be used.
* **`time_index`:** Time separation between extracted values. Can be e.g. `monthly` (default) or \
`yearly`.
**Common settings for all input options**
* **`drop_constants`:** Bool used to determine if constant parameters should be dropped. \
Default is True.
---
!> For smry data it is **strongly recommended** to keep the data frequency to a regular frequency \
(like `monthly` or `yearly`). This applies to both csv input and when reading from `UNSMRY` \
(controlled by the `sampling` key). This is because the statistics and fancharts are calculated \
per DATE over all realizations in an ensemble, and the available dates should therefore not \
differ between individual realizations of an ensemble.
?> Vectors that are identified as historical vectors (e.g. FOPTH is the history of FOPT) will \
be plotted together with their non-historical counterparts as reference lines.
**Using simulation time series data directly from `.UNSMRY` files**
!> Parameter values are extracted automatically from the `parameters.txt` files in the individual
realizations if you have defined `ensembles`, using the `fmu-ensemble` library.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
**Using aggregated data**
?> Aggregated data may speed up the build of the app, as processing of `UNSMRY` files can be \
slow for large models.
"""
# pylint: disable=too-many-arguments
def __init__(
self,
app,
ensembles: Optional[list] = None,
csvfile_parameters: pathlib.Path = None,
csvfile_smry: pathlib.Path = None,
time_index: str = "monthly",
column_keys: Optional[list] = None,
drop_constants: bool = True,
):
super().__init__()
WEBVIZ_ASSETS.add(
pathlib.Path(webviz_subsurface.__file__).parent / "_assets" /
"css" / "container.css")
self.theme = app.webviz_settings["theme"]
self.time_index = time_index
self.column_keys = column_keys
self.ensembles = ensembles
self.csvfile_parameters = csvfile_parameters
self.csvfile_smry = csvfile_smry
if ensembles is not None:
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.pmodel = ParametersModel(
dataframe=self.emodel.load_parameters(),
theme=self.theme,
drop_constants=drop_constants,
)
self.vmodel = SimulationTimeSeriesModel(
dataframe=self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys),
theme=self.theme,
)
elif self.csvfile_parameters is None:
raise ValueError(
"Either ensembles or csvfile_parameters must be specified")
else:
self.pmodel = ParametersModel(
dataframe=read_csv(csvfile_parameters),
theme=self.theme,
drop_constants=drop_constants,
)
if self.csvfile_smry is not None:
self.vmodel = SimulationTimeSeriesModel(
dataframe=read_csv(csvfile_smry),
theme=self.theme.plotly_theme)
else:
self.vmodel = None
self.set_callbacks(app)
@property
def layout(self) -> dcc.Tabs:
return main_view(parent=self)
def set_callbacks(self, app) -> None:
parameter_qc_controller(self, app)
if self.vmodel is not None:
parameter_response_controller(self, app)
def add_webvizstore(self):
store = []
if self.ensembles is not None:
store.extend(self.emodel.webvizstore)
else:
store.extend((
read_csv,
[
{
"csv_file": self.csvfile_parameters
},
],
))
if self.csvfile_smry is not None:
store.extend((
read_csv,
[
{
"csv_file": self.csvfile_smry
},
],
))
return store
class ParameterParallelCoordinates(WebvizPluginABC):
"""Visualizes parameters used in FMU ensembles side-by-side. Also supports response coloring.
Useful to investigate:
* Initial parameter distributions, and convergence of parameters over multiple iterations.
* Trends in relations between parameters and responses.
!> At least two parameters have to be selected to make the plot work.
---
**Three main options for input data: Aggregated, file per realization and read from UNSMRY.**
**Using aggregated data**
* **`parameter_csv`:** Aggregated csvfile for input parameters with `REAL` and `ENSEMBLE` columns \
(absolute path or relative to config file).
* **`response_csv`:** Aggregated csvfile for response parameters with `REAL` and `ENSEMBLE` \
columns (absolute path or relative to config file).
**Using a response file per realization**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
* **`response_file`:** Local (per realization) csv file for response parameters (Cannot be \
combined with `response_csv` and `parameter_csv`).
* Parameter values are extracted automatically from the `parameters.txt` files in the individual
realizations of your defined `ensembles`, using the `fmu-ensemble` library.
**Using simulation time series data directly from `UNSMRY` files as responses**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize. The lack of `response_file` \
implies that the input data should be time series data from simulation `.UNSMRY` \
files, read using `fmu-ensemble`.
* **`column_keys`:** (Optional) slist of simulation vectors to include as responses when reading \
from UNSMRY-files in the defined ensembles (default is all vectors). * can be \
used as wild card.
* **`sampling`:** (Optional) sampling frequency when reading simulation data directly from \
`.UNSMRY`-files (default is monthly).
* Parameter values are extracted automatically from the `parameters.txt` files in the individual
realizations of your defined `ensembles`, using the `fmu-ensemble` library.
?> The `UNSMRY` input method implies that the "DATE" vector will be used as a filter \
of type `single` (as defined below under `response_filters`).
**Using the plugin without responses**
It is possible to use the plugin with only parameter data, in that case set the option \
`no_responses` to True, and give either `ensembles` or `parameter_csv` as input as described \
above. Response coloring and filtering will then not be available.
**Common settings for responses**
All of these are optional, some have defaults seen in the code snippet below.
* **`response_filters`:** Optional dictionary of responses (columns in csv file or simulation \
vectors) that can be used as row filtering before aggregation. \
Valid options:
* `single`: Dropdown with single selection.
* `multi`: Dropdown with multiple selection.
* `range`: Slider with range selection.
* **`response_ignore`:** List of response (columns in csv or simulation vectors) to ignore \
(cannot use with response_include).
* **`response_include`:** List of response (columns in csv or simulation vectors) to include \
(cannot use with response_ignore).
* **`aggregation`:** How to aggregate responses per realization. Either `sum` or `mean`.
Parameter values are extracted automatically from the `parameters.txt` files in the individual
realizations of your defined `ensembles`, using the `fmu-ensemble` library.
---
?> Non-numerical (string-based) input parameters and responses are removed.
?> The responses will be aggregated per realization; meaning that if your filters do not reduce \
the response to a single value per realization in your data, the values will be aggregated \
accoording to your defined `aggregation`. If e.g. the response is a form of volume, \
and the filters are regions (or other subdivisions of the total volume), then `sum` would \
be a natural aggregation. If on the other hand the response is the pressures in the \
same volume, aggregation as `mean` over the subdivisions of the same volume \
would make more sense (though the pressures in this case would not be volume weighted means, \
and the aggregation would therefore likely be imprecise).
!> It is **strongly recommended** to keep the data frequency to a regular frequency (like \
`monthly` or `yearly`). This applies to both csv input and when reading from `UNSMRY` \
(controlled by the `sampling` key). This is because the statistics are calculated per DATE over \
all realizations in an ensemble, and the available dates should therefore not differ between \
individual realizations of an ensemble.
**Using aggregated data**
The `parameter_csv` file must have columns `REAL`, `ENSEMBLE` and the parameter columns.
The `response_csv` file must have columns `REAL`, `ENSEMBLE` and the response columns \
(and the columns to use as `response_filters`, if that option is used).
**Using a response file per realization**
Parameters are extracted automatically from the `parameters.txt` files in the individual
realizations, using the `fmu-ensemble` library.
The `response_file` must have the response columns (and the columns to use as `response_filters`, \
if that option is used).
**Using simulation time series data directly from `UNSMRY` files as responses**
Parameters are extracted automatically from the `parameters.txt` files in the individual
realizations, using the `fmu-ensemble` library.
Responses are extracted automatically from the `UNSMRY` files in the individual realizations,
using the `fmu-ensemble` library.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
"""
# pylint: disable=too-many-arguments,too-many-branches
def __init__(
self,
app,
ensembles: list = None,
parameter_csv: Path = None,
response_csv: Path = None,
response_file: str = None,
response_filters: dict = None,
response_ignore: list = None,
response_include: list = None,
parameter_ignore: list = None,
column_keys: list = None,
sampling: str = "monthly",
aggregation: str = "sum",
no_responses=False,
):
super().__init__()
self.parameter_csv = parameter_csv if parameter_csv else None
self.response_csv = response_csv if response_csv else None
self.response_file = response_file if response_file else None
self.response_filters = response_filters if response_filters else {}
self.response_ignore = response_ignore if response_ignore else None
self.parameter_ignore = parameter_ignore if parameter_ignore else None
self.column_keys = column_keys
self.time_index = sampling
self.aggregation = aggregation
self.no_responses = no_responses
if response_ignore and response_include:
raise ValueError(
'Incorrent argument. Either provide "response_include", '
'"response_ignore" or neither'
)
if parameter_csv:
if ensembles or response_file:
raise ValueError(
'Incorrect arguments. Either provide "parameter_csv" or '
'"ensembles and/or response_file".'
)
if not self.no_responses:
if self.response_csv:
self.responsedf = read_csv(self.response_csv)
else:
raise ValueError("Incorrect arguments. Missing response_csv.")
self.parameterdf = read_csv(self.parameter_csv)
elif ensembles:
if self.response_csv:
raise ValueError(
'Incorrect arguments. Either provide "response_csv" or '
'"ensembles and/or response_file".'
)
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]["scratch_ensembles"][
ens
]
for ens in ensembles
}
)
self.parameterdf = self.emodel.load_parameters()
if not self.no_responses:
if self.response_file:
self.responsedf = self.emodel.load_csv(csv_file=response_file)
else:
self.responsedf = self.emodel.load_smry(
time_index=self.time_index, column_keys=self.column_keys
)
self.response_filters["DATE"] = "single"
else:
raise ValueError(
"Incorrect arguments."
'You have to define at least "ensembles" or "parameter_csv".'
)
if not self.no_responses:
self.check_runs()
self.check_response_filters()
if response_ignore:
self.responsedf.drop(
response_ignore, errors="ignore", axis=1, inplace=True
)
if response_include:
self.responsedf.drop(
self.responsedf.columns.difference(
[
"REAL",
"ENSEMBLE",
*response_include,
*list(response_filters.keys()),
]
),
errors="ignore",
axis=1,
inplace=True,
)
if parameter_ignore:
self.parameterdf.drop(parameter_ignore, axis=1, inplace=True)
# Integer value for each ensemble to be used for ensemble colormap
# self.uuid("COLOR") used to mitigate risk of already having a column named "COLOR" in the
# DataFrame.
self.parameterdf[self.uuid("COLOR")] = self.parameterdf.apply(
lambda row: self.ensembles.index(row["ENSEMBLE"]), axis=1
)
self.theme = app.webviz_settings["theme"]
self.set_callbacks(app)
@property
def parameters(self):
"""Returns numerical input parameters"""
return list(
self.parameterdf.drop(["ENSEMBLE", "REAL", self.uuid("COLOR")], axis=1)
.apply(pd.to_numeric, errors="coerce")
.dropna(how="all", axis="columns")
.columns
)
@property
def responses(self):
"""Returns valid responses. Filters out non numerical columns,
and filterable columns."""
responses = list(
self.responsedf.drop(["ENSEMBLE", "REAL"], axis=1)
.apply(pd.to_numeric, errors="coerce")
.dropna(how="all", axis="columns")
.columns
)
return [p for p in responses if p not in self.response_filters.keys()]
@property
def ensembles(self):
"""Returns list of ensembles"""
return list(self.parameterdf["ENSEMBLE"].unique())
@property
def ens_colormap(self):
"""Returns a discrete colormap with one color per ensemble"""
colors = self.theme.plotly_theme["layout"]["colorway"]
colormap = []
for i in range(0, len(self.ensembles)):
colormap.append([i / len(self.ensembles), colors[i]])
colormap.append([(i + 1) / len(self.ensembles), colors[i]])
return colormap
def check_runs(self):
"""Check that input parameters and response files have
the same number of runs"""
for col in ["ENSEMBLE", "REAL"]:
if sorted(list(self.parameterdf[col].unique())) != sorted(
list(self.responsedf[col].unique())
):
raise ValueError("Parameter and response files have different runs")
def check_response_filters(self):
"""Check that provided response filters are valid"""
if self.response_filters:
for col_name, col_type in self.response_filters.items():
if col_name not in self.responsedf.columns:
raise ValueError(f"{col_name} is not in response file")
if col_type not in ["single", "multi", "range"]:
raise ValueError(
f"Filter type {col_type} for {col_name} is not valid."
)
def make_response_filters(self, filters):
"""Returns a list of active response filters"""
filteroptions = []
if filters:
for i, (col_name, col_type) in enumerate(self.response_filters.items()):
filteroptions.append(
{"name": col_name, "type": col_type, "values": filters[i]}
)
return filteroptions
@property
def response_layout(self):
"""Layout to display selectors for response filters"""
if self.no_responses:
return []
children = [
html.Span("Response:", style={"font-weight": "bold"}),
dcc.Dropdown(
id=self.uuid("responses"),
options=[{"label": ens, "value": ens} for ens in self.responses],
clearable=False,
value=self.responses[0],
style={"marginBottom": "20px"},
persistence=True,
persistence_type="session",
),
]
if self.response_filters is not None:
for col_name, col_type in self.response_filters.items():
values = list(self.responsedf[col_name].unique())
if col_type == "multi":
selector = wcc.Select(
id=self.uuid(f"filter-{col_name}"),
options=[{"label": val, "value": val} for val in values],
value=values,
multi=True,
size=min(20, len(values)),
persistence=True,
persistence_type="session",
)
elif col_type == "single":
selector = dcc.Dropdown(
id=self.uuid(f"filter-{col_name}"),
options=[{"label": val, "value": val} for val in values],
value=values[0],
multi=False,
clearable=False,
persistence=True,
persistence_type="session",
)
children.append(
html.Div(
children=[
html.Label(col_name),
selector,
]
)
)
return [
html.Div(
id=self.uuid("view_response"),
style={"display": "none"},
children=children,
),
]
@property
def control_layout(self):
"""Layout to select ensembles and parameters"""
mode_select = (
[]
if self.no_responses
else [
html.Label(
children=[
html.Span("Mode:", style={"font-weight": "bold"}),
dcc.RadioItems(
id=self.uuid("mode"),
options=[
{"label": "Ensemble", "value": "ensemble"},
{"label": "Response", "value": "response"},
],
value="ensemble",
labelStyle={"display": "inline-block"},
persistence=True,
persistence_type="session",
),
]
)
]
)
return mode_select + [
html.Span("Ensemble:", style={"font-weight": "bold"}),
wcc.Select(
id=self.uuid("ensembles"),
options=[{"label": ens, "value": ens} for ens in self.ensembles],
multi=True,
value=self.ensembles,
size=min(len(self.ensembles), 10),
persistence=True,
persistence_type="session",
),
html.Label(
children=[
html.Span(
"Parameters:",
id=self.uuid("parameters"),
style={
"font-weight": "bold",
},
),
dcc.RadioItems(
id=self.uuid("exclude_include"),
options=[
{"label": "Exclude", "value": "exc"},
{"label": "Include", "value": "inc"},
],
value="exc",
labelStyle={"display": "inline-block"},
style={"fontSize": ".80em"},
persistence=True,
persistence_type="session",
),
]
),
wcc.Select(
id=self.uuid("parameter-list"),
options=[{"label": ens, "value": ens} for ens in self.parameters],
multi=True,
size=min(len(self.parameters), 15),
value=[],
style={
"marginBottom": "20px",
"fontSize": ".80em",
"overflowX": "auto",
},
persistence=True,
persistence_type="session",
),
]
@property
def layout(self):
"""Main layout"""
return wcc.FlexBox(
id=self.uuid("layout"),
children=[
html.Div(
style={"flex": 1},
children=(self.control_layout + self.response_layout),
),
html.Div(
style={"flex": 3},
children=wcc.Graph(
id=self.uuid("parcoords"),
),
),
],
)
@property
def parcoord_inputs(self):
inputs = [
Input(self.uuid("ensembles"), "value"),
Input(self.uuid("exclude_include"), "value"),
Input(self.uuid("parameter-list"), "value"),
]
if not self.no_responses:
inputs.extend(
[
Input(self.uuid("mode"), "value"),
Input(self.uuid("responses"), "value"),
]
)
if self.response_filters is not None:
inputs.extend(
[
Input(self.uuid(f"filter-{col}"), "value")
for col in self.response_filters
]
)
return inputs
@staticmethod
def set_grid_layout(columns):
return {
"display": "grid",
"alignContent": "space-around",
"justifyContent": "space-between",
"gridTemplateColumns": f"{columns}",
}
def set_callbacks(self, app):
@app.callback(
Output(self.uuid("parcoords"), "figure"),
self.parcoord_inputs,
)
def _update_parcoord(ens, exc_inc, parameter_list, *opt_args):
"""Updates parallel coordinates plot
Filter dataframe for chosen ensembles and parameters
Call render_parcoord to render new figure
"""
# Ensure selected ensembles is a list
ens = ens if isinstance(ens, list) else [ens]
# Ensure selected parameters is a list
parameter_list = (
parameter_list if isinstance(parameter_list, list) else [parameter_list]
)
special_columns = ["ENSEMBLE", "REAL", self.uuid("COLOR")]
if exc_inc == "exc":
parameterdf = self.parameterdf.drop(parameter_list, axis=1)
elif exc_inc == "inc":
parameterdf = self.parameterdf[special_columns + parameter_list]
params = [
param
for param in parameterdf.columns
if param not in special_columns and param in self.parameters
]
mode = opt_args[0] if opt_args else "ensemble"
# Need a default response
response = ""
if mode == "response":
if len(ens) != 1:
# Need to wait for update of ensemble selector to multi=False
raise PreventUpdate
df = parameterdf.loc[self.parameterdf["ENSEMBLE"] == ens[0]]
response = opt_args[1]
response_filters = opt_args[2:] if len(opt_args) > 2 else {}
filteroptions = self.make_response_filters(response_filters)
responsedf = filter_and_sum_responses(
self.responsedf,
ens[0],
response,
filteroptions=filteroptions,
aggregation=self.aggregation,
)
# Renaming to make it clear in plot.
responsedf.rename(
columns={response: f"Response: {response}"}, inplace=True
)
df = pd.merge(responsedf, df, on=["REAL"]).drop(columns=special_columns)
else:
# Filter on ensembles (ens) and active parameters (params),
# adding the COLOR column to the columns to keep
df = self.parameterdf[self.parameterdf["ENSEMBLE"].isin(ens)][
params + [self.uuid("COLOR")]
]
return render_parcoord(
df,
self.theme,
self.ens_colormap,
self.uuid("COLOR"),
self.ensembles,
mode,
params,
response,
)
@app.callback(
[
Output(self.uuid("ensembles"), "multi"),
Output(self.uuid("ensembles"), "value"),
Output(self.uuid("view_response"), "style"),
],
[Input(self.uuid("mode"), "value")],
)
def _update_mode(mode: str):
if mode == "ensemble":
return True, self.ensembles, {"display": "none"}
if mode == "response":
return False, self.ensembles[0], {"display": "block"}
# The error should never occur
raise ValueError("ensemble and response are the only valid modes.")
def add_webvizstore(self):
functions = []
if self.parameter_csv:
functions.append(
(
read_csv,
[
{
"csv_file": self.parameter_csv,
}
],
)
)
if self.response_csv:
functions.append(
(
read_csv,
[
{
"csv_file": self.response_csv,
}
],
)
)
else:
functions.extend(self.emodel.webvizstore)
return functions
def __init__(
self,
app,
ensembles: list = None,
parameter_csv: Path = None,
response_csv: Path = None,
response_file: str = None,
response_filters: dict = None,
response_ignore: list = None,
response_include: list = None,
parameter_ignore: list = None,
column_keys: list = None,
sampling: str = "monthly",
aggregation: str = "sum",
no_responses=False,
):
super().__init__()
self.parameter_csv = parameter_csv if parameter_csv else None
self.response_csv = response_csv if response_csv else None
self.response_file = response_file if response_file else None
self.response_filters = response_filters if response_filters else {}
self.response_ignore = response_ignore if response_ignore else None
self.parameter_ignore = parameter_ignore if parameter_ignore else None
self.column_keys = column_keys
self.time_index = sampling
self.aggregation = aggregation
self.no_responses = no_responses
if response_ignore and response_include:
raise ValueError(
'Incorrent argument. Either provide "response_include", '
'"response_ignore" or neither'
)
if parameter_csv:
if ensembles or response_file:
raise ValueError(
'Incorrect arguments. Either provide "parameter_csv" or '
'"ensembles and/or response_file".'
)
if not self.no_responses:
if self.response_csv:
self.responsedf = read_csv(self.response_csv)
else:
raise ValueError("Incorrect arguments. Missing response_csv.")
self.parameterdf = read_csv(self.parameter_csv)
elif ensembles:
if self.response_csv:
raise ValueError(
'Incorrect arguments. Either provide "response_csv" or '
'"ensembles and/or response_file".'
)
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]["scratch_ensembles"][
ens
]
for ens in ensembles
}
)
self.parameterdf = self.emodel.load_parameters()
if not self.no_responses:
if self.response_file:
self.responsedf = self.emodel.load_csv(csv_file=response_file)
else:
self.responsedf = self.emodel.load_smry(
time_index=self.time_index, column_keys=self.column_keys
)
self.response_filters["DATE"] = "single"
else:
raise ValueError(
"Incorrect arguments."
'You have to define at least "ensembles" or "parameter_csv".'
)
if not self.no_responses:
self.check_runs()
self.check_response_filters()
if response_ignore:
self.responsedf.drop(
response_ignore, errors="ignore", axis=1, inplace=True
)
if response_include:
self.responsedf.drop(
self.responsedf.columns.difference(
[
"REAL",
"ENSEMBLE",
*response_include,
*list(response_filters.keys()),
]
),
errors="ignore",
axis=1,
inplace=True,
)
if parameter_ignore:
self.parameterdf.drop(parameter_ignore, axis=1, inplace=True)
# Integer value for each ensemble to be used for ensemble colormap
# self.uuid("COLOR") used to mitigate risk of already having a column named "COLOR" in the
# DataFrame.
self.parameterdf[self.uuid("COLOR")] = self.parameterdf.apply(
lambda row: self.ensembles.index(row["ENSEMBLE"]), axis=1
)
self.theme = app.webviz_settings["theme"]
self.set_callbacks(app)
class ParameterResponseCorrelation(WebvizPluginABC):
"""Visualizes correlations between numerical input parameters and responses.
---
**Three main options for input data: Aggregated, file per realization and read from UNSMRY.**
**Using aggregated data**
* **`parameter_csv`:** Aggregated csvfile for input parameters with `REAL` and `ENSEMBLE` columns \
(absolute path or relative to config file).
* **`response_csv`:** Aggregated csvfile for response parameters with `REAL` and `ENSEMBLE` \
columns (absolute path or relative to config file).
**Using a response file per realization**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
* **`response_file`:** Local (per realization) csv file for response parameters (Cannot be \
combined with `response_csv` and `parameter_csv`).
**Using simulation time series data directly from `UNSMRY` files as responses**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize. The lack of `response_file` \
implies that the input data should be time series data from simulation `.UNSMRY` \
files, read using `fmu-ensemble`.
* **`column_keys`:** (Optional) slist of simulation vectors to include as responses when reading \
from UNSMRY-files in the defined ensembles (default is all vectors). * can be \
used as wild card.
* **`sampling`:** (Optional) sampling frequency when reading simulation data directly from \
`.UNSMRY`-files (default is monthly).
?> The `UNSMRY` input method implies that the "DATE" vector will be used as a filter \
of type `single` (as defined below under `response_filters`).
**Common settings for all input options**
All of these are optional, some have defaults seen in the code snippet below.
* **`response_filters`:** Optional dictionary of responses (columns in csv file or simulation \
vectors) that can be used as row filtering before aggregation. \
Valid options:
* `single`: Dropdown with single selection.
* `multi`: Dropdown with multiple selection.
* `range`: Slider with range selection.
* **`response_ignore`:** List of response (columns in csv or simulation vectors) to ignore \
(cannot use with response_include).
* **`response_include`:** List of response (columns in csv or simulation vectors) to include \
(cannot use with response_ignore).
* **`aggregation`:** How to aggregate responses per realization. Either `sum` or `mean`.
* **`corr_method`:** Correlation method. Either `pearson` or `spearman`.
---
?> Non-numerical (string-based) input parameters and responses are removed.
?> The responses will be aggregated per realization; meaning that if your filters do not reduce \
the response to a single value per realization in your data, the values will be aggregated \
accoording to your defined `aggregation`. If e.g. the response is a form of volume, \
and the filters are regions (or other subdivisions of the total volume), then `sum` would \
be a natural aggregation. If on the other hand the response is the pressures in the \
same volume, aggregation as `mean` over the subdivisions of the same volume \
would make more sense (though the pressures in this case would not be volume weighted means, \
and the aggregation would therefore likely be imprecise).
!> It is **strongly recommended** to keep the data frequency to a regular frequency (like \
`monthly` or `yearly`). This applies to both csv input and when reading from `UNSMRY` \
(controlled by the `sampling` key). This is because the statistics are calculated per DATE over \
all realizations in an ensemble, and the available dates should therefore not differ between \
individual realizations of an ensemble.
**Using aggregated data**
The `parameter_csv` file must have columns `REAL`, `ENSEMBLE` and the parameter columns.
The `response_csv` file must have columns `REAL`, `ENSEMBLE` and the response columns \
(and the columns to use as `response_filters`, if that option is used).
**Using a response file per realization**
Parameters are extracted automatically from the `parameters.txt` files in the individual
realizations, using the `fmu-ensemble` library.
The `response_file` must have the response columns (and the columns to use as `response_filters`, \
if that option is used).
**Using simulation time series data directly from `UNSMRY` files as responses**
Parameters are extracted automatically from the `parameters.txt` files in the individual
realizations, using the `fmu-ensemble` library.
Responses are extracted automatically from the `UNSMRY` files in the individual realizations,
using the `fmu-ensemble` library.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
"""
# pylint:disable=too-many-arguments
def __init__(
self,
app,
parameter_csv: Path = None,
response_csv: Path = None,
ensembles: list = None,
response_file: str = None,
response_filters: dict = None,
response_ignore: list = None,
response_include: list = None,
column_keys: list = None,
sampling: str = "monthly",
aggregation: str = "sum",
corr_method: str = "pearson",
):
super().__init__()
self.parameter_csv = parameter_csv if parameter_csv else None
self.response_csv = response_csv if response_csv else None
self.response_file = response_file if response_file else None
self.response_filters = response_filters if response_filters else {}
self.response_ignore = response_ignore if response_ignore else None
self.column_keys = column_keys
self.time_index = sampling
self.corr_method = corr_method
self.aggregation = aggregation
if response_ignore and response_include:
raise ValueError(
'Incorrent argument. either provide "response_include", '
'"response_ignore" or neither')
if parameter_csv and response_csv:
if ensembles or response_file:
raise ValueError(
'Incorrect arguments. Either provide "csv files" or '
'"ensembles and response_file".')
self.parameterdf = read_csv(self.parameter_csv)
self.responsedf = read_csv(self.response_csv)
elif ensembles:
self.ens_paths = {
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
}
self.parameterdf = load_parameters(ensemble_paths=self.ens_paths,
ensemble_set_name="EnsembleSet")
if self.response_file:
self.responsedf = load_csv(
ensemble_paths=self.ens_paths,
csv_file=response_file,
ensemble_set_name="EnsembleSet",
)
else:
self.emodel = EnsembleSetModel(ensemble_paths=self.ens_paths)
self.responsedf = self.emodel.load_smry(
column_keys=self.column_keys,
time_index=self.time_index,
)
self.response_filters["DATE"] = "single"
else:
raise ValueError(
'Incorrect arguments. Either provide "csv files" or "ensembles and response_file".'
)
self.check_runs()
self.check_response_filters()
if response_ignore:
self.responsedf.drop(response_ignore,
errors="ignore",
axis=1,
inplace=True)
if response_include:
self.responsedf.drop(
self.responsedf.columns.difference([
"REAL",
"ENSEMBLE",
*response_include,
*list(response_filters.keys()),
]),
errors="ignore",
axis=1,
inplace=True,
)
self.plotly_theme = app.webviz_settings["theme"].plotly_theme
self.uid = uuid4()
self.set_callbacks(app)
def ids(self, element):
"""Generate unique id for dom element"""
return f"{element}-id-{self.uid}"
@property
def tour_steps(self):
steps = [
{
"id":
self.ids("layout"),
"content":
("Dashboard displaying correlation between selected "
"response and input parameters."),
},
{
"id":
self.ids("correlation-graph"),
"content":
("Visualization of the correlations between currently selected "
"response and input parameters ranked by the absolute correlation "
"coefficient. Click on any correlation to visualize the distribution "
"between that parameter and the response."),
},
{
"id":
self.ids("distribution-graph"),
"content":
("Visualized the distribution of the response and the selected input parameter "
"in the correlation chart."),
},
{
"id": self.ids("ensemble"),
"content": ("Select the active ensemble."),
},
{
"id": self.ids("responses"),
"content": ("Select the active response."),
},
]
return steps
@property
def responses(self):
"""Returns valid responses. Filters out non numerical columns,
and filterable columns"""
responses = list(
self.responsedf.drop(["ENSEMBLE", "REAL"], axis=1).apply(
pd.to_numeric, errors="coerce").dropna(how="all",
axis="columns").columns)
return [p for p in responses if p not in self.response_filters.keys()]
@property
def parameters(self):
"""Returns numerical input parameters"""
parameters = list(
self.parameterdf.drop(["ENSEMBLE", "REAL"], axis=1).apply(
pd.to_numeric, errors="coerce").dropna(how="all",
axis="columns").columns)
return parameters
@property
def ensembles(self):
"""Returns list of ensembles"""
return list(self.parameterdf["ENSEMBLE"].unique())
def check_runs(self):
"""Check that input parameters and response files have
the same number of runs"""
for col in ["ENSEMBLE", "REAL"]:
if sorted(list(self.parameterdf[col].unique())) != sorted(
list(self.responsedf[col].unique())):
raise ValueError(
"Parameter and response files have different runs")
def check_response_filters(self):
"""'Check that provided response filters are valid"""
if self.response_filters:
for col_name, col_type in self.response_filters.items():
if col_name not in self.responsedf.columns:
raise ValueError(f"{col_name} is not in response file")
if col_type not in ["single", "multi", "range"]:
raise ValueError(
f"Filter type {col_type} for {col_name} is not valid.")
@property
def filter_layout(self):
"""Layout to display selectors for response filters"""
children = []
for col_name, col_type in self.response_filters.items():
domid = self.ids(f"filter-{col_name}")
values = list(self.responsedf[col_name].unique())
if col_type == "multi":
selector = wcc.Select(
id=domid,
options=[{
"label": val,
"value": val
} for val in values],
value=values,
multi=True,
size=min(20, len(values)),
persistence=True,
persistence_type="session",
)
elif col_type == "single":
selector = dcc.Dropdown(
id=domid,
options=[{
"label": val,
"value": val
} for val in values],
value=values[0],
multi=False,
clearable=False,
persistence=True,
persistence_type="session",
)
elif col_type == "range":
selector = make_range_slider(domid, self.responsedf[col_name],
col_name)
else:
return children
children.append(
html.Div(children=[
html.Label(col_name),
selector,
]))
return children
@property
def control_layout(self):
"""Layout to select e.g. iteration and response"""
return [
html.Div([
html.Label("Ensemble"),
dcc.Dropdown(
id=self.ids("ensemble"),
options=[{
"label": ens,
"value": ens
} for ens in self.ensembles],
clearable=False,
value=self.ensembles[0],
persistence=True,
persistence_type="session",
),
]),
html.Div([
html.Label("Response"),
dcc.Dropdown(
id=self.ids("responses"),
options=[{
"label": ens,
"value": ens
} for ens in self.responses],
clearable=False,
value=self.responses[0],
persistence=True,
persistence_type="session",
),
]),
]
@property
def layout(self):
"""Main layout"""
return wcc.FlexBox(
id=self.ids("layout"),
children=[
html.Div(
style={"flex": 3},
children=[
wcc.Graph(self.ids("correlation-graph")),
dcc.Store(id=self.ids("initial-parameter"),
storage_type="session"),
],
),
html.Div(
style={"flex": 3},
children=wcc.Graph(self.ids("distribution-graph")),
),
html.Div(
style={"flex": 1},
children=self.control_layout +
self.filter_layout if self.response_filters else [],
),
],
)
@property
def correlation_input_callbacks(self):
"""List of Inputs for correlation callback"""
callbacks = [
Input(self.ids("ensemble"), "value"),
Input(self.ids("responses"), "value"),
]
if self.response_filters:
for col_name in self.response_filters:
callbacks.append(Input(self.ids(f"filter-{col_name}"),
"value"))
return callbacks
@property
def distribution_input_callbacks(self):
"""List of Inputs for distribution callback"""
callbacks = [
Input(self.ids("correlation-graph"), "clickData"),
Input(self.ids("initial-parameter"), "data"),
Input(self.ids("ensemble"), "value"),
Input(self.ids("responses"), "value"),
]
if self.response_filters:
for col_name in self.response_filters:
callbacks.append(Input(self.ids(f"filter-{col_name}"),
"value"))
return callbacks
def make_response_filters(self, filters):
"""Returns a list of active response filters"""
filteroptions = []
if filters:
for i, (col_name,
col_type) in enumerate(self.response_filters.items()):
filteroptions.append({
"name": col_name,
"type": col_type,
"values": filters[i]
})
return filteroptions
def set_callbacks(self, app):
@app.callback(
[
Output(self.ids("correlation-graph"), "figure"),
Output(self.ids("initial-parameter"), "data"),
],
self.correlation_input_callbacks,
)
def _update_correlation_graph(ensemble, response, *filters):
"""Callback to update correlation graph
1. Filters and aggregates response dataframe per realization
2. Filters parameters dataframe on selected ensemble
3. Merge parameter and response dataframe
4. Correlate merged dataframe
5. Sort correlation for selected response by absolute values
6. Remove nan values return correlation graph
"""
filteroptions = self.make_response_filters(filters)
responsedf = filter_and_sum_responses(
self.responsedf,
ensemble,
response,
filteroptions=filteroptions,
aggregation=self.aggregation,
)
parameterdf = self.parameterdf.loc[self.parameterdf["ENSEMBLE"] ==
ensemble]
df = pd.merge(responsedf, parameterdf, on=["REAL"])
corrdf = correlate(df, response=response, method=self.corr_method)
try:
corr_response = (corrdf[response].dropna().drop(
["REAL", response], axis=0))
return (
make_correlation_plot(corr_response, response,
self.plotly_theme, self.corr_method),
corr_response.index[-1],
)
except KeyError:
return (
{
"layout": {
"title":
"<b>Cannot calculate correlation for given selection</b><br>"
"Select a different response or filter setting."
}
},
None,
)
@app.callback(
Output(self.ids("distribution-graph"), "figure"),
self.distribution_input_callbacks,
)
def _update_distribution_graph(clickdata, initial_parameter, ensemble,
response, *filters):
"""Callback to update distribution graphs.
1. Filters and aggregates response dataframe per realization
2. Filters parameters dataframe on selected ensemble
3. Merge parameter and response dataframe
4. Generate scatterplot and histograms
"""
if clickdata:
parameter = clickdata["points"][0]["y"]
elif initial_parameter:
parameter = initial_parameter
else:
return {}
filteroptions = self.make_response_filters(filters)
responsedf = filter_and_sum_responses(
self.responsedf,
ensemble,
response,
filteroptions=filteroptions,
aggregation=self.aggregation,
)
parameterdf = self.parameterdf.loc[self.parameterdf["ENSEMBLE"] ==
ensemble]
df = pd.merge(responsedf, parameterdf,
on=["REAL"])[["REAL", parameter, response]]
return make_distribution_plot(df, parameter, response,
self.plotly_theme)
def add_webvizstore(self):
if self.parameter_csv and self.response_csv:
return [
(
read_csv,
[{
"csv_file": self.parameter_csv,
}],
),
(
read_csv,
[{
"csv_file": self.response_csv,
}],
),
]
functions = [
(
load_parameters,
[{
"ensemble_paths": self.ens_paths,
"ensemble_set_name": "EnsembleSet",
}],
),
]
if self.response_file:
functions.append((
load_csv,
[{
"ensemble_paths": self.ens_paths,
"csv_file": self.response_file,
"ensemble_set_name": "EnsembleSet",
}],
))
else:
functions.extend(self.emodel.webvizstore)
return functions
def __init__(
self,
app,
parameter_csv: Path = None,
response_csv: Path = None,
ensembles: list = None,
response_file: str = None,
response_filters: dict = None,
response_ignore: list = None,
response_include: list = None,
column_keys: list = None,
sampling: str = "monthly",
aggregation: str = "sum",
corr_method: str = "pearson",
):
super().__init__()
self.parameter_csv = parameter_csv if parameter_csv else None
self.response_csv = response_csv if response_csv else None
self.response_file = response_file if response_file else None
self.response_filters = response_filters if response_filters else {}
self.response_ignore = response_ignore if response_ignore else None
self.column_keys = column_keys
self.time_index = sampling
self.corr_method = corr_method
self.aggregation = aggregation
if response_ignore and response_include:
raise ValueError(
'Incorrent argument. either provide "response_include", '
'"response_ignore" or neither')
if parameter_csv and response_csv:
if ensembles or response_file:
raise ValueError(
'Incorrect arguments. Either provide "csv files" or '
'"ensembles and response_file".')
self.parameterdf = read_csv(self.parameter_csv)
self.responsedf = read_csv(self.response_csv)
elif ensembles:
self.ens_paths = {
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
}
self.parameterdf = load_parameters(ensemble_paths=self.ens_paths,
ensemble_set_name="EnsembleSet")
if self.response_file:
self.responsedf = load_csv(
ensemble_paths=self.ens_paths,
csv_file=response_file,
ensemble_set_name="EnsembleSet",
)
else:
self.emodel = EnsembleSetModel(ensemble_paths=self.ens_paths)
self.responsedf = self.emodel.load_smry(
column_keys=self.column_keys,
time_index=self.time_index,
)
self.response_filters["DATE"] = "single"
else:
raise ValueError(
'Incorrect arguments. Either provide "csv files" or "ensembles and response_file".'
)
self.check_runs()
self.check_response_filters()
if response_ignore:
self.responsedf.drop(response_ignore,
errors="ignore",
axis=1,
inplace=True)
if response_include:
self.responsedf.drop(
self.responsedf.columns.difference([
"REAL",
"ENSEMBLE",
*response_include,
*list(response_filters.keys()),
]),
errors="ignore",
axis=1,
inplace=True,
)
self.plotly_theme = app.webviz_settings["theme"].plotly_theme
self.uid = uuid4()
self.set_callbacks(app)
class BhpQc(WebvizPluginABC):
"""QC simulated bottom hole pressures (BHP) from reservoir simulations.
Can be used to check if your simulated BHPs are in a realistic range.
E.g. check if your simulated bottom hole pressures are very low in producers,
or very high injectors.
---
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
---
Data is read directly from the UNSMRY files with the raw frequency (not resampled).
Resampling and csvs are not supported to avoid potential of interpolation, which
might cover extreme BHP values.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
"""
def __init__(
self,
app: dash.Dash,
ensembles: list,
wells: Optional[List[str]] = None,
):
super().__init__()
if wells is None:
self.column_keys = ["WBHP:*"]
else:
self.column_keys = [f"WBHP:{well}" for well in wells]
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.smry = self.emodel.load_smry(time_index="raw",
column_keys=self.column_keys)
self.theme = app.webviz_settings["theme"]
self.set_callbacks(app)
@property
def tour_steps(self) -> List[dict]:
return [
{
"id":
self.uuid("layout"),
"content":
("Dashboard for BHP QC:"
"Check that simulated bottom hole pressures are realistic."),
},
{
"id": self.uuid("ensemble"),
"content": "Select ensemble to QC."
},
{
"id": self.uuid("sort_by"),
"content": "Sort wells left to right according to this value.",
},
{
"id":
self.uuid("n_wells"),
"content":
("Show max selected number of top ranked wells after sorting and filtering."
),
},
{
"id": self.uuid("wells"),
"content": "Filter wells."
},
]
@property
def ensembles(self) -> List[str]:
return list(self.smry["ENSEMBLE"].unique())
@property
def wells(self) -> List[set]:
return sorted(
list(
set(col[5:] for col in self.smry.columns
if col.startswith("WBHP:"))))
@property
def ens_colors(self) -> dict:
return unique_colors(self.ensembles, self.theme)
@property
def label_map(self) -> Dict[str, str]:
return {
"Mean": "mean",
"Count (data points)": "count",
"Stddev": "std",
"Minimum": "min",
"Maximum": "max",
"P10 (high)": "high_p10",
"P50": "p50",
"P90 (low)": "low_p90",
}
@property
def layout(self) -> wcc.FlexBox:
return wcc.FlexBox(
id=self.uuid("layout"),
children=[
html.Div(
style={"flex": 1},
children=[
html.Label(children=[
html.Span(
"Ensemble:",
style={"font-weight": "bold"},
),
dcc.Dropdown(
id=self.uuid("ensemble"),
options=[{
"label": i,
"value": i
} for i in self.ensembles],
value=self.ensembles[0],
clearable=False,
multi=False,
),
], ),
html.Label(children=[
html.Span("Plot type:",
style={"font-weight": "bold"}),
dcc.Dropdown(
id=self.uuid("plot_type"),
options=[{
"label": i,
"value": i
} for i in [
"Fan chart",
"Bar chart",
"Line chart",
]],
clearable=False,
value="Fan chart",
),
], ),
html.Label(
id=self.uuid("select_stat"),
style={"display": "none"},
children=[
html.Span("Select statistics:",
style={"font-weight": "bold"}),
wcc.Select(
id=self.uuid("stat_bars"),
options=[{
"label": key,
"value": value
} for key, value in self.label_map.items()
],
size=8,
value=["count", "low_p90", "p50"],
),
],
),
html.Label(children=[
html.Span("Sort by:",
style={"font-weight": "bold"}),
dcc.Dropdown(
id=self.uuid("sort_by"),
options=[{
"label": key,
"value": value
} for key, value in self.label_map.items()],
clearable=False,
value="low_p90",
),
], ),
dcc.RadioItems(
id=self.uuid("ascending"),
options=[
{
"label": "Ascending",
"value": True
},
{
"label": "Descending",
"value": False
},
],
value=True,
labelStyle={"display": "inline-block"},
),
html.Label(children=[
html.Span(
"Max number of wells in plot:",
style={"font-weight": "bold"},
),
dcc.Slider(
id=self.uuid("n_wells"),
min=1,
max=len(self.wells),
value=min(10, len(self.wells)),
marks={
1: 1,
len(self.wells): len(self.wells)
},
),
]),
html.Label(children=[
html.Span("Wells:", style={"font-weight": "bold"}),
wcc.Select(
id=self.uuid("wells"),
options=[{
"label": i,
"value": i
} for i in self.wells],
size=min([len(self.wells), 20]),
value=self.wells,
),
], ),
],
),
html.Div(
style={"flex": 3},
children=[
html.Div(
# style={"height": "300px"},
children=wcc.Graph(id=self.uuid("graph")), ),
],
),
],
)
def set_callbacks(self, app: dash.Dash) -> None:
@app.callback(
Output(self.uuid("graph"), "figure"),
Input(self.uuid("ensemble"), "value"),
Input(self.uuid("plot_type"), "value"),
Input(self.uuid("n_wells"), "value"),
Input(self.uuid("wells"), "value"),
Input(self.uuid("sort_by"), "value"),
Input(self.uuid("stat_bars"), "value"),
Input(self.uuid("ascending"), "value"),
)
def _update_graph(
ensemble: str,
plot_type: str,
n_wells: int,
wells: Union[str, List[str]],
sort_by: str,
stat_bars: Union[str, List[str]],
ascending: bool,
) -> dict:
wells = wells if isinstance(wells, list) else [wells]
stat_bars = stat_bars if isinstance(stat_bars,
list) else [stat_bars]
df = filter_df(df=self.smry, ensemble=ensemble, wells=wells)
stat_df = (calc_statistics(df).sort_values(
sort_by, ascending=ascending).iloc[0:n_wells, :])
traces = []
if plot_type == "Fan chart":
traces.extend(
add_fanchart_traces(
ens_stat_df=stat_df,
color=self.ens_colors[ensemble],
legend_group=ensemble,
))
elif plot_type in ["Bar chart", "Line chart"]:
for stat in stat_bars:
yaxis = "y2" if stat == "count" else "y"
if plot_type == "Bar chart":
traces.append({
"x":
[vec[5:] for vec in stat_df.index], # strip WBHP:
"y":
stat_df[stat],
"name": [
key for key, value in self.label_map.items()
if value == stat
][0],
"yaxis":
yaxis,
"type":
"bar",
"offsetgroup":
stat,
"showlegend":
True,
})
elif plot_type == "Line chart":
traces.append({
"x":
[vec[5:] for vec in stat_df.index], # strip WBHP:
"y":
stat_df[stat],
"name": [
key for key, value in self.label_map.items()
if value == stat
][0],
"yaxis":
yaxis,
"type":
"line",
"offsetgroup":
stat,
"showlegend":
True,
})
else:
raise ValueError("Invalid plot type.")
layout = self.theme.create_themed_layout({
"yaxis": {
"side": "left",
"title": "Bottom hole pressure",
"showgrid": False,
},
"yaxis2": {
"side": "right",
"overlaying": "y",
"title": "Count (data points)",
"showgrid": False,
},
"xaxis": {
"showgrid": False
},
"barmode": "group",
"legend": {
"x": 1.05
},
})
return {"data": traces, "layout": layout}
@app.callback(
Output(self.uuid("select_stat"), "style"),
Input(self.uuid("plot_type"), "value"),
)
def _update_stat_selector(plot_type: str) -> dict:
return ({
"display": "none"
} if plot_type == "Fan chart" else {
"display": "block"
})
def add_webvizstore(self) -> List[Tuple[Callable, list]]:
return self.emodel.webvizstore
class ReservoirSimulationTimeSeries(WebvizPluginABC):
"""Visualizes reservoir simulation time series data for FMU ensembles.
**Features**
* Visualization of realization time series as line charts.
* Visualization of ensemble time series statistics as fan charts.
* Visualization of single date ensemble statistics as histograms.
* Calculation and visualization of delta ensembles.
* Calculation and visualization of average rates and cumulatives over a specified time interval.
* Download of visualized data to csv files (except histogram data).
---
**Two main options for input data: Aggregated and read from UNSMRY.**
**Using aggregated data**
* **`csvfile`:** Aggregated csv file with `REAL`, `ENSEMBLE`, \
`DATE` and vector columns.
**Using simulation time series data directly from `UNSMRY` files**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
* **`column_keys`:** List of vectors to extract. If not given, all vectors \
from the simulations will be extracted. Wild card asterisk `*` can be used.
* **`sampling`:** Time separation between extracted values. Can be e.g. `monthly` (default) or \
`yearly`.
**Common optional settings for both input options**
* **`obsfile`**: File with observations to plot together with the relevant time series. \
(absolute path or relative to config file).
* **`options`:** Options to initialize plots with:
* `vector1` : First vector to display
* `vector2` : Second vector to display
* `vector3` : Third vector to display
* `visualization` : `realizations`, `statistics` or `statistics_hist`
* `date` : Date to show in histograms
* **`line_shape_fallback`:** Fallback interpolation method between points. Vectors identified as \
rates or phase ratios are always backfilled, vectors identified as cumulative (totals) are \
always linearly interpolated. The rest use the fallback.
Supported options:
* `linear` (default)
* `backfilled`
* `hv`, `vh`, `hvh`, `vhv` and `spline` (regular Plotly options).
---
?> Vectors that are identified as historical vectors (e.g. FOPTH is the history of FOPT) will \
be plotted together with their non-historical counterparts as reference lines, and they are \
therefore not selectable as vectors to plot initially.
?> The `obsfile` is a common (optional) file for all ensembles, which currently has to be made \
manually. [An example of the format can be found here]\
(https://github.com/equinor/webviz-subsurface-testdata/blob/master/\
reek_history_match/share/observations/observations.yml).
!> It is **strongly recommended** to keep the data frequency to a regular frequency (like \
`monthly` or `yearly`). This applies to both csv input and when reading from `UNSMRY` \
(controlled by the `sampling` key). This is because the statistics and fancharts are calculated \
per DATE over all realizations in an ensemble, and the available dates should therefore not \
differ between individual realizations of an ensemble.
**Using aggregated data**
The `csvfile` must have columns `ENSEMBLE`, `REAL` and `DATE` in addition to the individual
vectors.
* [Example of aggregated file]\
(https://github.com/equinor/webviz-subsurface-testdata/blob/master/aggregated_data/smry.csv).
**Using simulation time series data directly from `.UNSMRY` files**
Vectors are extracted automatically from the `UNSMRY` files in the individual realizations,
using the `fmu-ensemble` library.
?> Using the `UNSMRY` method will also extract metadata like units, and whether the vector is a \
rate, a cumulative, or historical. Units are e.g. added to the plot titles, while rates and \
cumulatives are used to decide the line shapes in the plot. Aggregated data may on the other \
speed up the build of the app, as processing of `UNSMRY` files can be slow for large models. \
Using this method is required to use the average rate and interval cumulative functionalities, \
as they require identification of vectors that are cumulatives.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
"""
ENSEMBLE_COLUMNS = ["REAL", "ENSEMBLE", "DATE"]
# pylint:disable=too-many-arguments
def __init__(
self,
app: dash.Dash,
csvfile: Path = None,
ensembles: list = None,
obsfile: Path = None,
column_keys: list = None,
sampling: str = "monthly",
options: dict = None,
line_shape_fallback: str = "linear",
):
super().__init__()
self.csvfile = csvfile
self.obsfile = obsfile
self.time_index = sampling
self.column_keys = column_keys
if csvfile and ensembles:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile" or "ensembles"'
)
self.observations = {}
if obsfile:
self.observations = check_and_format_observations(
get_path(self.obsfile))
self.smry: pd.DataFrame
self.smry_meta: Union[pd.DataFrame, None]
if csvfile:
self.smry = read_csv(csvfile)
self.smry_meta = None
# Check of time_index for data to use in resampling. Quite naive as it only checks for
# unique values of the DATE column, and not per realization.
#
# Currently not necessary as we don't allow resampling for average rates and intervals
# unless we have metadata, which csvfile input currently doesn't support.
# See: https://github.com/equinor/webviz-subsurface/issues/402
self.time_index = pd.infer_freq(
sorted(pd.to_datetime(self.smry["DATE"]).unique()))
elif ensembles:
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.smry = self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys)
self.smry_meta = self.emodel.load_smry_meta(
column_keys=self.column_keys, )
else:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile" or "ensembles"'
)
if any(
[col.startswith(("AVG_", "INTVL_")) for col in self.smry.columns]):
raise ValueError(
"Your data set includes time series vectors which have names starting with"
"'AVG_' and/or 'INTVL_'. These prefixes are not allowed, as they are used"
"internally in the plugin.")
self.smry_cols = [
c for c in self.smry.columns
if c not in ReservoirSimulationTimeSeries.ENSEMBLE_COLUMNS
and not historical_vector(c, self.smry_meta,
False) in self.smry.columns
]
self.dropdown_options = []
for vec in self.smry_cols:
self.dropdown_options.append({
"label": f"{simulation_vector_description(vec)} ({vec})",
"value": vec
})
if (self.smry_meta is not None and self.smry_meta.is_total[vec]
and self.time_index is not None):
# Get the likely name for equivalent rate vector and make dropdown options.
# Requires that the time_index was either defined or possible to infer.
avgrate_vec = rename_vec_from_cum(vector=vec, as_rate=True)
interval_vec = rename_vec_from_cum(vector=vec, as_rate=False)
self.dropdown_options.append({
"label":
f"{simulation_vector_description(avgrate_vec)} ({avgrate_vec})",
"value": avgrate_vec,
})
self.dropdown_options.append({
"label":
f"{simulation_vector_description(interval_vec)} ({interval_vec})",
"value": interval_vec,
})
self.ensembles = list(self.smry["ENSEMBLE"].unique())
self.theme = app.webviz_settings["theme"]
self.plot_options = options if options else {}
self.plot_options["date"] = (str(self.plot_options.get("date")) if
self.plot_options.get("date") else None)
self.line_shape_fallback = set_simulation_line_shape_fallback(
line_shape_fallback)
# Check if initially plotted vectors exist in data, raise ValueError if not.
missing_vectors = [
value for key, value in self.plot_options.items()
if key in ["vector1", "vector2", "vector3"]
and value not in self.smry_cols
]
if missing_vectors:
raise ValueError(
f"Cannot find: {', '.join(missing_vectors)} to plot initially in "
"ReservoirSimulationTimeSeries. Check that the vectors exist in your data, and "
"that they are not missing in a non-default column_keys list in the yaml config "
"file.")
self.allow_delta = len(self.ensembles) > 1
self.set_callbacks(app)
@property
def ens_colors(self) -> dict:
return unique_colors(self.ensembles, self.theme)
@property
def tour_steps(self) -> List[dict]:
return [
{
"id":
self.uuid("layout"),
"content":
"Dashboard displaying reservoir simulation time series.",
},
{
"id":
self.uuid("graph"),
"content":
("Visualization of selected time series. "
"Different options can be set in the menu to the left."),
},
{
"id":
self.uuid("ensemble"),
"content":
("Display time series from one or several ensembles. "
"Different ensembles will be overlain in the same plot."),
},
{
"id":
self.uuid("vectors"),
"content":
("Display up to three different time series. "
"Each time series will be visualized in a separate plot. "
"Vectors prefixed with AVG_ and INTVL_ are calculated in the fly "
"from cumulative vectors, providing average rates and interval cumulatives "
"over a time interval that can be defined in the menu."),
},
{
"id":
self.uuid("visualization"),
"content":
("Choose between different visualizations. 1. Show time series as "
"individual lines per realization. 2. Show statistical fanchart per "
"ensemble. 3. Show statistical fanchart per ensemble and histogram "
"per date. Select a data by clicking in the plot."),
},
{
"id":
self.uuid("cum_interval"),
"content":
("Defines the time interval the average rates (prefixed AVG_) and interval "
"cumulatives (prefixed INTVL_) are calculated over. Disabled unless at least "
"one time series dependent on the interval setting is chosen."
"The option might be completely hidden if the data input does not support "
"calculation from cumulatives."),
},
]
@staticmethod
def set_grid_layout(columns: str, padding: int = 0) -> Dict[str, str]:
return {
"display": "grid",
"alignContent": "space-around",
"justifyContent": "space-between",
"gridTemplateColumns": f"{columns}",
"padding": f"{padding}px",
}
@property
def time_interval_options(self) -> List[str]:
if self.time_index == "daily":
return ["daily", "monthly", "yearly"]
if self.time_index == "monthly":
return ["monthly", "yearly"]
if self.time_index == "yearly":
return ["yearly"]
return []
@property
def delta_layout(self) -> html.Div:
show_delta = "block" if self.allow_delta else "none"
return html.Div(children=[
html.Div(
style={"display": show_delta},
children=html.Label(children=[
html.Span("Mode:", style={"font-weight": "bold"}),
dcc.RadioItems(
id=self.uuid("mode"),
style={"marginBottom": "25px"},
options=[
{
"label": "Individual ensembles",
"value": "ensembles",
},
{
"label": "Delta between ensembles",
"value": "delta_ensembles",
},
],
value="ensembles",
persistence=True,
persistence_type="session",
),
]),
),
html.Div(
id=self.uuid("show_ensembles"),
children=html.Label(children=[
html.Span("Selected ensembles:",
style={"font-weight": "bold"}),
dcc.Dropdown(
id=self.uuid("ensemble"),
clearable=False,
multi=True,
options=[{
"label": i,
"value": i
} for i in self.ensembles],
value=[self.ensembles[0]],
persistence=True,
persistence_type="session",
),
], ),
),
html.Div(
id=self.uuid("calc_delta"),
style={"display": "none"},
children=[
html.Span(
"Selected ensemble delta (A-B):",
style={"font-weight": "bold"},
),
html.Div(
style=self.set_grid_layout("1fr 1fr"),
children=[
html.Div([
html.Label(
style={"fontSize": "12px"},
children="Ensemble A",
),
dcc.Dropdown(
id=self.uuid("base_ens"),
clearable=False,
options=[{
"label": i,
"value": i
} for i in self.ensembles],
value=self.ensembles[0],
persistence=True,
persistence_type="session",
),
]),
html.Div([
html.Label(
style={"fontSize": "12px"},
children="Ensemble B",
),
dcc.Dropdown(
id=self.uuid("delta_ens"),
clearable=False,
options=[{
"label": i,
"value": i
} for i in self.ensembles],
value=self.ensembles[-1],
persistence=True,
persistence_type="session",
),
]),
],
),
],
),
])
@property
def from_cumulatives_layout(self) -> html.Div:
return html.Div(
style=({
"marginTop": "25px",
"display": "block"
} if len(self.time_interval_options) > 0
and self.smry_meta is not None else {
"display": "none"
}),
children=[
html.Span(
"Calculated from cumulatives:",
style={"font-weight": "bold"},
),
html.Div(
"Average (AVG_) and interval (INTVL_) time series",
style={
"font-style": "italic",
"font-size": "0.75em"
},
),
html.Div(
dcc.RadioItems(
id=self.uuid("cum_interval"),
options=[{
"label": (f"{i.lower().capitalize()}"),
"value": i,
"disabled": False,
} for i in self.time_interval_options],
value=self.time_index,
persistence=True,
persistence_type="session",
), ),
],
)
@property
def layout(self) -> wcc.FlexBox:
return wcc.FlexBox(
id=self.uuid("layout"),
children=[
html.Div(
style={"flex": 1},
children=[
self.delta_layout,
html.Div(
id=self.uuid("vectors"),
style={"marginTop": "25px"},
children=[
html.Span("Time series:",
style={"font-weight": "bold"}),
dcc.Dropdown(
style={
"marginTop": "5px",
"marginBottom": "5px",
"fontSize": ".95em",
},
optionHeight=55,
id=self.uuid("vector1"),
clearable=False,
multi=False,
options=self.dropdown_options,
value=self.plot_options.get(
"vector1", self.smry_cols[0]),
persistence=True,
persistence_type="session",
),
dcc.Dropdown(
style={
"marginBottom": "5px",
"fontSize": ".95em"
},
optionHeight=55,
id=self.uuid("vector2"),
clearable=True,
multi=False,
placeholder="Add additional series",
options=self.dropdown_options,
value=self.plot_options.get(
"vector2", None),
persistence=True,
persistence_type="session",
),
dcc.Dropdown(
style={"fontSize": ".95em"},
optionHeight=55,
id=self.uuid("vector3"),
clearable=True,
multi=False,
placeholder="Add additional series",
options=self.dropdown_options,
value=self.plot_options.get(
"vector3", None),
persistence=True,
persistence_type="session",
),
],
),
html.Div(
id=self.uuid("visualization"),
style={"marginTop": "25px"},
children=[
html.Span("Visualization:",
style={"font-weight": "bold"}),
dcc.RadioItems(
id=self.uuid("statistics"),
options=[
{
"label": "Individual realizations",
"value": "realizations",
},
{
"label": "Statistical fanchart",
"value": "statistics",
},
{
"label":
"Statistical fanchart and histogram",
"value": "statistics_hist",
},
],
value=self.plot_options.get(
"visualization", "statistics"),
persistence=True,
persistence_type="session",
),
],
),
self.from_cumulatives_layout,
],
),
html.Div(
style={"flex": 3},
children=[
html.Div(children=wcc.Graph(
id=self.uuid("graph"), ), ),
dcc.Store(
id=self.uuid("date"),
storage_type="session",
data=json.dumps(self.plot_options.get(
"date", None)),
),
],
),
],
)
# pylint: disable=too-many-statements
def set_callbacks(self, app: dash.Dash) -> None:
@app.callback(
Output(self.uuid("graph"), "figure"),
[
Input(self.uuid("vector1"), "value"),
Input(self.uuid("vector2"), "value"),
Input(self.uuid("vector3"), "value"),
Input(self.uuid("ensemble"), "value"),
Input(self.uuid("mode"), "value"),
Input(self.uuid("base_ens"), "value"),
Input(self.uuid("delta_ens"), "value"),
Input(self.uuid("statistics"), "value"),
Input(self.uuid("cum_interval"), "value"),
Input(self.uuid("date"), "data"),
],
)
# pylint: disable=too-many-instance-attributes, too-many-arguments, too-many-locals, too-many-branches
def _update_graph(
vector1: str,
vector2: Union[str, None],
vector3: Union[str, None],
ensembles: List[str],
calc_mode: str,
base_ens: str,
delta_ens: str,
visualization: str,
cum_interval: str,
stored_date: str,
) -> dict:
"""Callback to update all graphs based on selections"""
if not isinstance(ensembles, list):
raise TypeError("ensembles should always be of type list")
if calc_mode not in ["ensembles", "delta_ensembles"]:
raise PreventUpdate
# Combine selected vectors
vectors = [vector1]
if vector2:
vectors.append(vector2)
if vector3:
vectors.append(vector3)
# Synthesize ensembles list for delta mode
if calc_mode == "delta_ensembles":
ensembles = [base_ens, delta_ens]
# Retrieve previous/current selected date
date = json.loads(stored_date) if stored_date else None
# Titles for subplots
# TODO(Sigurd)
# Added None to union since date_to_interval_conversion() may return None.
# Need input on what should be done since a None title is probably not what we want
titles: List[Union[str, None]] = []
for vec in vectors:
if sys.version_info >= (3, 9):
unit_vec = vec.removeprefix("AVG_").removeprefix("INTVL_")
else:
unit_vec = (vec[4:] if vec.startswith("AVG_") else
(vec[6:] if vec.startswith("INTVL_") else vec))
if self.smry_meta is None:
titles.append(simulation_vector_description(vec))
else:
titles.append(
f"{simulation_vector_description(vec)}"
f" [{simulation_unit_reformat(self.smry_meta.unit[unit_vec])}]"
)
if visualization == "statistics_hist":
titles.append(
date_to_interval_conversion(date=date,
vector=vec,
interval=cum_interval,
as_date=False))
# Make a plotly subplot figure
fig = make_subplots(
rows=len(vectors),
cols=2 if visualization == "statistics_hist" else 1,
shared_xaxes=True,
vertical_spacing=0.05,
subplot_titles=titles,
)
# Loop through each vector and calculate relevant plot
legends = []
dfs = calculate_vector_dataframes(
smry=self.smry,
smry_meta=self.smry_meta,
ensembles=ensembles,
vectors=vectors,
calc_mode=calc_mode,
visualization=visualization,
time_index=self.time_index,
cum_interval=cum_interval,
)
for i, vector in enumerate(vectors):
if dfs[vector]["data"].empty:
continue
line_shape = get_simulation_line_shape(
line_shape_fallback=self.line_shape_fallback,
vector=vector,
smry_meta=self.smry_meta,
)
if visualization in ["statistics", "statistics_hist"]:
traces = add_statistic_traces(
dfs[vector]["stat"],
vector,
colors=self.ens_colors,
line_shape=line_shape,
interval=cum_interval,
)
if visualization == "statistics_hist":
histdata = add_histogram_traces(
dfs[vector]["data"],
vector,
date=date,
colors=self.ens_colors,
interval=cum_interval,
)
for trace in histdata:
fig.add_trace(trace, i + 1, 2)
elif visualization == "realizations":
traces = add_realization_traces(
dfs[vector]["data"],
vector,
colors=self.ens_colors,
line_shape=line_shape,
interval=cum_interval,
)
else:
raise PreventUpdate
historical_vector_name = historical_vector(
vector=vector, smry_meta=self.smry_meta)
if (historical_vector_name and historical_vector_name
in dfs[vector]["data"].columns):
traces.append(
add_history_trace(
dfs[vector]["data"],
historical_vector_name,
line_shape,
))
# Remove unwanted legends(only keep one for each ensemble)
for trace in traces:
if trace.get("showlegend"):
if trace.get("legendgroup") in legends:
trace["showlegend"] = False
else:
legends.append(trace.get("legendgroup"))
fig.add_trace(trace, i + 1, 1)
# Add observations
if calc_mode != "delta_ensembles" and self.observations.get(
vector):
for trace in add_observation_trace(
self.observations.get(vector)):
fig.add_trace(trace, i + 1, 1)
fig = fig.to_dict()
fig["layout"].update(
height=800,
margin={
"t": 20,
"b": 0
},
barmode="overlay",
bargap=0.01,
bargroupgap=0.2,
)
fig["layout"] = self.theme.create_themed_layout(fig["layout"])
if visualization == "statistics_hist":
# Remove linked x-axis for histograms
if "xaxis2" in fig["layout"]:
fig["layout"]["xaxis2"]["matches"] = None
fig["layout"]["xaxis2"]["showticklabels"] = True
if "xaxis4" in fig["layout"]:
fig["layout"]["xaxis4"]["matches"] = None
fig["layout"]["xaxis4"]["showticklabels"] = True
if "xaxis6" in fig["layout"]:
fig["layout"]["xaxis6"]["matches"] = None
fig["layout"]["xaxis6"]["showticklabels"] = True
return fig
@app.callback(
self.plugin_data_output,
[self.plugin_data_requested],
[
State(self.uuid("vector1"), "value"),
State(self.uuid("vector2"), "value"),
State(self.uuid("vector3"), "value"),
State(self.uuid("ensemble"), "value"),
State(self.uuid("mode"), "value"),
State(self.uuid("base_ens"), "value"),
State(self.uuid("delta_ens"), "value"),
State(self.uuid("statistics"), "value"),
State(self.uuid("cum_interval"), "value"),
],
)
def _user_download_data(
data_requested: Union[int, None],
vector1: str,
vector2: Union[str, None],
vector3: Union[str, None],
ensembles: List[str],
calc_mode: str,
base_ens: str,
delta_ens: str,
visualization: str,
cum_interval: str,
) -> Union[EncodedFile, str]:
"""Callback to download data based on selections"""
# Combine selected vectors
vectors = [vector1]
if vector2:
vectors.append(vector2)
if vector3:
vectors.append(vector3)
# Ensure selected ensembles is a list and prevent update if invalid calc_mode
if calc_mode == "delta_ensembles":
ensembles = [base_ens, delta_ens]
elif calc_mode == "ensembles":
if not isinstance(ensembles, list):
raise TypeError("ensembles should always be of type list")
else:
raise PreventUpdate
dfs = calculate_vector_dataframes(
smry=self.smry,
smry_meta=self.smry_meta,
ensembles=ensembles,
vectors=vectors,
calc_mode=calc_mode,
visualization=visualization,
time_index=self.time_index,
cum_interval=cum_interval,
)
for vector, df in dfs.items():
if visualization in ["statistics", "statistics_hist"]:
dfs[vector]["stat"] = df["stat"].sort_values(
by=[("", "ENSEMBLE"), ("", "DATE")])
if vector.startswith(("AVG_", "INTVL_")):
dfs[vector]["stat"]["", "DATE"] = dfs[vector]["stat"][
"", "DATE"].astype(str)
dfs[vector]["stat"]["", "DATE"] = dfs[vector]["stat"][
"", "DATE"].apply(
date_to_interval_conversion,
vector=vector,
interval=cum_interval,
as_date=False,
)
else:
dfs[vector]["data"] = df["data"].sort_values(
by=["ENSEMBLE", "REAL", "DATE"])
# Reorder columns
dfs[vector]["data"] = dfs[vector]["data"][
["ENSEMBLE", "REAL", "DATE"] + [
col for col in dfs[vector]["data"].columns
if col not in ["ENSEMBLE", "REAL", "DATE"]
]]
if vector.startswith(("AVG_", "INTVL_")):
dfs[vector]["data"]["DATE"] = dfs[vector]["data"][
"DATE"].astype(str)
dfs[vector]["data"]["DATE"] = dfs[vector]["data"][
"DATE"].apply(
date_to_interval_conversion,
vector=vector,
interval=cum_interval,
as_date=False,
)
# : is replaced with _ in filenames to stay within POSIX portable pathnames
# (e.g. : is not valid in a Windows path)
return (WebvizPluginABC.plugin_data_compress(
[{
"filename": f"{vector.replace(':', '_')}.csv",
"content": df.get("stat", df["data"]).to_csv(index=False),
} for vector, df in dfs.items()]) if data_requested else "")
@app.callback(
[
Output(self.uuid("show_ensembles"), "style"),
Output(self.uuid("calc_delta"), "style"),
],
[Input(self.uuid("mode"), "value")],
)
def _update_mode(mode: str) -> Tuple[dict, dict]:
"""Switch displayed ensemble selector for delta/no-delta"""
if mode == "ensembles":
style = {"display": "block"}, {"display": "none"}
else:
style = {"display": "none"}, {"display": "block"}
return style
@app.callback(
Output(self.uuid("date"), "data"),
[Input(self.uuid("graph"), "clickData")],
[State(self.uuid("date"), "data")],
)
def _update_date(clickdata: dict, date: str) -> str:
"""Store clicked date for use in other callback"""
date = clickdata["points"][0]["x"] if clickdata else json.loads(
date)
return json.dumps(date)
@app.callback(
Output(self.uuid("cum_interval"), "options"),
[
Input(self.uuid("vector1"), "value"),
Input(self.uuid("vector2"), "value"),
Input(self.uuid("vector3"), "value"),
],
[State(self.uuid("cum_interval"), "options")],
)
def _activate_interval_radio_buttons(
vector1: str,
vector2: Union[str, None],
vector3: Union[str, None],
options: List[dict],
) -> List[dict]:
"""Switch activate/deactivate radio buttons for selectibg interval for
calculations from cumulatives"""
active = False
for vector in [vector1, vector2, vector3]:
if vector is not None and vector.startswith(
("AVG_", "INTVL_")):
active = True
break
if active:
return [
dict(option, **{"disabled": False}) for option in options
]
return [dict(option, **{"disabled": True}) for option in options]
def add_webvizstore(self) -> List[Tuple[Callable, list]]:
functions: List[Tuple[Callable, list]] = []
if self.csvfile:
functions.append((read_csv, [{"csv_file": self.csvfile}]))
else:
functions.extend(self.emodel.webvizstore)
if self.obsfile:
functions.append((get_path, [{"path": self.obsfile}]))
return functions
def __init__(
self,
app: dash.Dash,
csvfile: Path = None,
ensembles: list = None,
obsfile: Path = None,
column_keys: list = None,
sampling: str = "monthly",
options: dict = None,
line_shape_fallback: str = "linear",
):
super().__init__()
self.csvfile = csvfile
self.obsfile = obsfile
self.time_index = sampling
self.column_keys = column_keys
if csvfile and ensembles:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile" or "ensembles"'
)
self.observations = {}
if obsfile:
self.observations = check_and_format_observations(
get_path(self.obsfile))
self.smry: pd.DataFrame
self.smry_meta: Union[pd.DataFrame, None]
if csvfile:
self.smry = read_csv(csvfile)
self.smry_meta = None
# Check of time_index for data to use in resampling. Quite naive as it only checks for
# unique values of the DATE column, and not per realization.
#
# Currently not necessary as we don't allow resampling for average rates and intervals
# unless we have metadata, which csvfile input currently doesn't support.
# See: https://github.com/equinor/webviz-subsurface/issues/402
self.time_index = pd.infer_freq(
sorted(pd.to_datetime(self.smry["DATE"]).unique()))
elif ensembles:
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.smry = self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys)
self.smry_meta = self.emodel.load_smry_meta(
column_keys=self.column_keys, )
else:
raise ValueError(
'Incorrent arguments. Either provide a "csvfile" or "ensembles"'
)
if any(
[col.startswith(("AVG_", "INTVL_")) for col in self.smry.columns]):
raise ValueError(
"Your data set includes time series vectors which have names starting with"
"'AVG_' and/or 'INTVL_'. These prefixes are not allowed, as they are used"
"internally in the plugin.")
self.smry_cols = [
c for c in self.smry.columns
if c not in ReservoirSimulationTimeSeries.ENSEMBLE_COLUMNS
and not historical_vector(c, self.smry_meta,
False) in self.smry.columns
]
self.dropdown_options = []
for vec in self.smry_cols:
self.dropdown_options.append({
"label": f"{simulation_vector_description(vec)} ({vec})",
"value": vec
})
if (self.smry_meta is not None and self.smry_meta.is_total[vec]
and self.time_index is not None):
# Get the likely name for equivalent rate vector and make dropdown options.
# Requires that the time_index was either defined or possible to infer.
avgrate_vec = rename_vec_from_cum(vector=vec, as_rate=True)
interval_vec = rename_vec_from_cum(vector=vec, as_rate=False)
self.dropdown_options.append({
"label":
f"{simulation_vector_description(avgrate_vec)} ({avgrate_vec})",
"value": avgrate_vec,
})
self.dropdown_options.append({
"label":
f"{simulation_vector_description(interval_vec)} ({interval_vec})",
"value": interval_vec,
})
self.ensembles = list(self.smry["ENSEMBLE"].unique())
self.theme = app.webviz_settings["theme"]
self.plot_options = options if options else {}
self.plot_options["date"] = (str(self.plot_options.get("date")) if
self.plot_options.get("date") else None)
self.line_shape_fallback = set_simulation_line_shape_fallback(
line_shape_fallback)
# Check if initially plotted vectors exist in data, raise ValueError if not.
missing_vectors = [
value for key, value in self.plot_options.items()
if key in ["vector1", "vector2", "vector3"]
and value not in self.smry_cols
]
if missing_vectors:
raise ValueError(
f"Cannot find: {', '.join(missing_vectors)} to plot initially in "
"ReservoirSimulationTimeSeries. Check that the vectors exist in your data, and "
"that they are not missing in a non-default column_keys list in the yaml config "
"file.")
self.allow_delta = len(self.ensembles) > 1
self.set_callbacks(app)
class PropertyStatistics(WebvizPluginABC):
"""This plugin visualizes ensemble statistics calculated from grid properties.
---
**The main input to this plugin is property statistics extracted from grid models.
See the documentation in [fmu-tools](http://fmu-docs.equinor.com/) on how to generate this data.
Additional data includes UNSMRY data and optionally irap binary surfaces stored in standardized \
FMU format.
**Input data can be provided in two ways: Aggregated or read from ensembles stored on scratch.**
**Using aggregated data**
* **`csvfile_smry`:** Aggregated `csv` file for volumes with `REAL`, `ENSEMBLE`, `DATE` and \
vector columns (absolute path or relative to config file).
* **`csvfile_statistics`:** Aggregated `csv` file for property statistics. See the \
documentation in [fmu-tools](http://fmu-docs.equinor.com/) on how to generate this data.
**Using raw ensemble data stored in realization folders**
* **`ensembles`:** Which ensembles in `shared_settings` to visualize.
* **`statistic_file`:** Csv file for each realization with property statistics.
* **`column_keys`:** List of vectors to extract. If not given, all vectors \
from the simulations will be extracted. Wild card asterisk `*` can be used.
* **`time_index`:** Time separation between extracted values. Can be e.g. `monthly` (default) or \
`yearly`.
* **`surface_renaming`:** Optional dictionary to rename properties/zones to match filenames \
stored on FMU standardized format (zone--property.gri)
---
?> Folders with statistical surfaces are assumed located at \
`<ensemble_path>/share/results/maps/<ensemble>/<statistic>` where `statistic` are subfolders \
with statistical calculation: `mean`, `stddev`, `p10`, `p90`, `min`, `max`.
!> Surface data is currently not available when using aggregated files.
!> For smry data it is **strongly recommended** to keep the data frequency to a regular frequency \
(like `monthly` or `yearly`). This applies to both csv input and when reading from `UNSMRY` \
(controlled by the `sampling` key). This is because the statistics and fancharts are calculated \
per DATE over all realizations in an ensemble, and the available dates should therefore not \
differ between individual realizations of an ensemble.
**Using aggregated data**
**Using simulation time series data directly from `.UNSMRY` files**
Time series data are extracted automatically from the `UNSMRY` files in the individual
realizations, using the `fmu-ensemble` library.
?> Using the `UNSMRY` method will also extract metadata like units, and whether the vector is a \
rate, a cumulative, or historical. Units are e.g. added to the plot titles, while rates and \
cumulatives are used to decide the line shapes in the plot. Aggregated data may on the other \
speed up the build of the app, as processing of `UNSMRY` files can be slow for large models.
!> The `UNSMRY` files are auto-detected by `fmu-ensemble` in the `eclipse/model` folder of the \
individual realizations. You should therefore not have more than one `UNSMRY` file in this \
folder, to avoid risk of not extracting the right data.
"""
# pylint: disable=too-many-arguments
def __init__(
self,
app: dash.Dash,
ensembles: Optional[list] = None,
statistics_file: str = "share/results/tables/gridpropstatistics.csv",
csvfile_statistics: pathlib.Path = None,
csvfile_smry: pathlib.Path = None,
surface_renaming: Optional[dict] = None,
time_index: str = "monthly",
column_keys: Optional[list] = None,
):
super().__init__()
WEBVIZ_ASSETS.add(
pathlib.Path(webviz_subsurface.__file__).parent / "_assets" /
"css" / "container.css")
# TODO(Sigurd) fix this once we get a separate webviz_settings parameter
self.theme: WebvizConfigTheme = app.webviz_settings["theme"]
self.time_index = time_index
self.column_keys = column_keys
self.statistics_file = statistics_file
self.ensembles = ensembles
self.csvfile_statistics = csvfile_statistics
self.csvfile_smry = csvfile_smry
self.surface_folders: Union[dict, None]
if ensembles is not None:
self.emodel = EnsembleSetModel(
ensemble_paths={
ens: app.webviz_settings["shared_settings"]
["scratch_ensembles"][ens]
for ens in ensembles
})
self.pmodel = PropertyStatisticsModel(
dataframe=self.emodel.load_csv(
csv_file=pathlib.Path(self.statistics_file)),
theme=self.theme,
)
self.vmodel = SimulationTimeSeriesModel(
dataframe=self.emodel.load_smry(time_index=self.time_index,
column_keys=self.column_keys),
theme=self.theme,
)
self.surface_folders = {
ens: folder / "share" / "results" / "maps" / ens
for ens, folder in self.emodel.ens_folders.items()
}
else:
self.pmodel = PropertyStatisticsModel(
dataframe=read_csv(csvfile_statistics), theme=self.theme)
self.vmodel = SimulationTimeSeriesModel(
dataframe=read_csv(csvfile_smry), theme=self.theme)
self.surface_folders = None
self.surface_renaming = surface_renaming if surface_renaming else {}
self.set_callbacks(app)
@property
def layout(self) -> dcc.Tabs:
return main_view(parent=self)
def set_callbacks(self, app: dash.Dash) -> None:
property_qc_controller(self, app)
if len(self.pmodel.ensembles) > 1:
property_delta_controller(self, app)
property_response_controller(self, app)
def add_webvizstore(self) -> List[Tuple[Callable, list]]:
store: List[Tuple[Callable, list]] = []
if self.ensembles is not None:
store.extend(self.emodel.webvizstore)
else:
store.extend([(
read_csv,
[
{
"csv_file": self.csvfile_smry
},
{
"csv_file": self.csvfile_statistics
},
],
)])
if self.surface_folders is not None:
store.extend(surface_store(parent=self))
return store