def test_dump_csv_line(self):
ecl_sum_vector = EclSumKeyWordVector(self.ecl_sum)
ecl_sum_vector.addKeywords("F*")
with self.assertRaises(KeyError):
ecl_sum_vector.addKeyword("MISSING")
dtime = datetime.datetime(2002, 1, 1, 0, 0, 0)
with TestAreaContext("EclSum/csv_dump"):
test_file_name = self.createTestPath("dump.csv")
outputH = copen(test_file_name, "w")
self.ecl_sum.dumpCSVLine(dtime, ecl_sum_vector, outputH)
assert os.path.isfile(test_file_name)
def _load_smry_into_table(smry_filename: str) -> pa.Table:
"""
Reads data from SMRY file into PyArrow Table.
DATE column is stored as an Arrow timetamp with ms resolution, timestamp[ms]
All numeric columns will be stored as 32 bit float
Summary meta data will be attached per field/column of the table's schema under the
'smry_meta' key
"""
eclsum = EclSum(smry_filename, include_restart=False, lazy_load=False)
# For now, we go via a set to prune out duplicate entries being returned by EclSumKeyWordVector,
# see: https://github.com/equinor/ecl/issues/816#issuecomment-865881283
column_names: List[str] = list(
set(EclSumKeyWordVector(eclsum, add_keywords=True)))
# Exclude CPI columns from export
org_col_count = len(column_names)
column_names = [
colname for colname in column_names if not _is_cpi_column(colname)
]
if len(column_names) != org_col_count:
logger.info(
f"Excluding {org_col_count - len(column_names)} CPI columns from export"
)
# Fetch the dates as a numpy array with ms resolution
np_dates_ms = eclsum.numpy_dates
smry_meta_dict = _create_smry_meta_dict(eclsum, column_names)
# Datatypes to use for DATE column and all the numeric columns
dt_timestamp_ms = pa.timestamp("ms")
dt_float32 = pa.float32()
# Build schema for the table
field_list: List[pa.Field] = []
field_list.append(pa.field("DATE", dt_timestamp_ms))
for colname in column_names:
field_metadata = {b"smry_meta": json.dumps(smry_meta_dict[colname])}
field_list.append(
pa.field(colname, dt_float32, metadata=field_metadata))
schema = pa.schema(field_list)
# Now extract all the summary vectors one by one
# We do this through EclSum.numpy_vector() instead of EclSum.pandas_frame() since
# the latter throws an exception if the SMRY data has timestamps beyond 2262,
# see: https://github.com/equinor/ecl/issues/802
column_arrays = [np_dates_ms]
for colname in column_names:
colvector = eclsum.numpy_vector(colname)
column_arrays.append(colvector)
table = pa.table(column_arrays, schema=schema)
return table
def test_kw_vector(self):
case1 = create_case()
case2 = createEclSum("CSV" , [("FOPR", None , 0, "SM3/DAY") , ("FOPT" , None , 0, "SM3"), ("FWPT" , None , 0, "SM3")],
sim_length_days = 100,
num_report_step = 10,
num_mini_step = 10,
func_table = {"FOPT" : fopt,
"FOPR" : fopr ,
"FWPT" : fgpt })
kw_list = EclSumKeyWordVector( case1 )
kw_list.add_keyword("FOPT")
kw_list.add_keyword("FGPT")
kw_list.add_keyword("FOPR")
t = case1.getDataStartTime( ) + datetime.timedelta( days = 43 );
data = case1.get_interp_row( kw_list , t )
for d1,d2 in zip(data, [ case1.get_interp("FOPT", date = t),
case1.get_interp("FOPT", date = t),
case1.get_interp("FOPT", date = t) ]):
self.assertFloatEqual(d1,d2)
tmp = []
for key in kw_list:
tmp.append(key)
for (k1,k2) in zip(kw_list,tmp):
self.assertEqual(k1,k2)
kw_list2 = kw_list.copy(case2)
self.assertIn("FOPT", kw_list2)
self.assertIn("FOPR", kw_list2)
self.assertIn("FGPT", kw_list2)
data2 = case2.get_interp_row( kw_list2 , t )
self.assertEqual(len(data2), 3)
self.assertEqual(data[0], data2[0])
self.assertEqual(data[2], data2[2])
with TestAreaContext("sum_vector"):
with cwrap.open("f1.txt","w") as f:
case1.dumpCSVLine(t, kw_list, f)
with cwrap.open("f2.txt", "w") as f:
case2.dumpCSVLine(t,kw_list2,f)
with open("f1.txt") as f:
d1 = f.readline().split(",")
with open("f2.txt") as f:
d2 = f.readline().split(",")
self.assertEqual(d1[0],d2[0])
self.assertEqual(d1[2],d2[2])
self.assertEqual(d2[1],"")
def test_kw_vector(self):
case1 = create_case()
case2 = createEclSum("CSV" , [("FOPR", None , 0, "SM3/DAY") , ("FOPT" , None , 0, "SM3"), ("FWPT" , None , 0, "SM3")],
sim_length_days = 100,
num_report_step = 10,
num_mini_step = 10,
func_table = {"FOPT" : fopt,
"FOPR" : fopr ,
"FWPT" : fgpt })
kw_list = EclSumKeyWordVector( case1 )
kw_list.add_keyword("FOPT")
kw_list.add_keyword("FGPT")
kw_list.add_keyword("FOPR")
t = case1.getDataStartTime( ) + datetime.timedelta( days = 43 );
data = case1.get_interp_row( kw_list , t )
for d1,d2 in zip(data, [ case1.get_interp("FOPT", date = t),
case1.get_interp("FOPT", date = t),
case1.get_interp("FOPT", date = t) ]):
self.assertFloatEqual(d1,d2)
tmp = []
for key in kw_list:
tmp.append(key)
for (k1,k2) in zip(kw_list,tmp):
self.assertEqual(k1,k2)
kw_list2 = kw_list.copy(case2)
self.assertIn("FOPT", kw_list2)
self.assertIn("FOPR", kw_list2)
self.assertIn("FGPT", kw_list2)
data2 = case2.get_interp_row( kw_list2 , t )
self.assertEqual(len(data2), 3)
self.assertEqual(data[0], data2[0])
self.assertEqual(data[2], data2[2])
with TestAreaContext("sum_vector"):
with cwrap.open("f1.txt","w") as f:
case1.dumpCSVLine(t, kw_list, f)
with cwrap.open("f2.txt", "w") as f:
case2.dumpCSVLine(t,kw_list2,f)
with open("f1.txt") as f:
d1 = f.readline().split(",")
with open("f2.txt") as f:
d2 = f.readline().split(",")
self.assertEqual(d1[0],d2[0])
self.assertEqual(d1[2],d2[2])
self.assertEqual(d2[1],"")
def test_dump_csv_line(self):
ecl_sum_vector = EclSumKeyWordVector(self.ecl_sum)
ecl_sum_vector.addKeywords("F*")
with self.assertRaises(KeyError):
ecl_sum_vector.addKeyword("MISSING")
dtime = datetime.datetime(2002, 1, 1, 0, 0, 0)
with TestAreaContext("EclSum/csv_dump"):
test_file_name = self.createTestPath("dump.csv")
outputH = copen(test_file_name, "w")
self.ecl_sum.dumpCSVLine(dtime, ecl_sum_vector, outputH)
assert os.path.isfile(test_file_name)
def _libecl_eclsum_pandas_frame(
eclsum: EclSum,
time_index: Optional[Union[List[dt.date], List[dt.datetime]]] = None,
column_keys: Optional[List[str]] = None,
) -> pd.DataFrame:
"""Build a Pandas dataframe from an EclSum object.
Temporarily copied from libecl to circumvent bug
https://github.com/equinor/ecl/issues/802
"""
if column_keys is None:
keywords = EclSumKeyWordVector(eclsum, add_keywords=True)
else:
keywords = EclSumKeyWordVector(eclsum)
for key in column_keys:
keywords.add_keywords(key)
if len(keywords) == 0:
raise ValueError("No valid key")
# pylint: disable=protected-access
if time_index is None:
time_index = eclsum.dates # Changed from libecl
data = np.zeros([len(time_index), len(keywords)])
EclSum._init_pandas_frame(
eclsum, keywords, data.ctypes.data_as(ctypes.POINTER(ctypes.c_double))
)
else:
time_points = eclsum._make_time_vector(time_index)
data = np.zeros([len(time_points), len(keywords)])
EclSum._init_pandas_frame_interp(
eclsum,
keywords,
time_points,
data.ctypes.data_as(ctypes.POINTER(ctypes.c_double)),
)
# Do not give datetime64[ms] to Pandas, it will try to convert it
# to datetime64[ns] and error hard if it is out of bounds (year 2262)
assert isinstance(time_index[0], (dt.date, dt.datetime))
frame = pd.DataFrame(
index=time_index,
columns=list(keywords),
data=data,
)
# frame.index.type is now either datetime64[ns] or datetime.datetime (object)
# depending on whether the date range ended before 2262.
return frame
def _extract_well_connection_status(filename: Path) -> pd.DataFrame:
# pylint: disable=too-many-locals
"""Exctracts well connection status history for each compdat connection that
is included in the summary data on the form CPI:WELL,I,J,K.
From the CPI time series it is possible to extract the status of the connection
because it is 0 when the connection is SHUT and >0 when the connection is open.
The output from this function is one row for every time a connection changes
status. The earliest date for any connection will be OPEN, i.e a cell can not
be SHUT before it has been OPEN. This means that any cells that are always SHUT
will not be included in the export.
"""
eclsum = EclSum(str(filename), include_restart=False, lazy_load=False)
column_names: Set[str] = set(EclSumKeyWordVector(eclsum,
add_keywords=True))
np_dates_ms = eclsum.numpy_dates
cpi_columns = [
col for col in column_names
if re.match("^CPI:[A-Z0-9_-]{1,8}:[0-9]+,[0-9]+,[0-9]+$", col)
]
df = pd.DataFrame(columns=["DATE", "WELL", "I", "J", "K", "OP/SH"])
for col in cpi_columns:
colsplit = col.split(":")
well = colsplit[1]
i, j, k = colsplit[2].split(",")
vector = eclsum.numpy_vector(col)
status_changes = _get_status_changes(np_dates_ms, vector)
for date, status in status_changes:
df.loc[df.shape[0]] = [date, well, i, j, k, status]
return df
