def read(self, items=None, time_steps=None): """ Read data from a dfs0 file. Parameters ---------- items: list[int] or list[str], optional Read only selected items, by number (0-based), or by name time_steps: int or list[int], optional Read only selected time_steps Returns ------- Dataset A dataset with data dimensions [t] """ if not os.path.exists(self._filename): raise FileNotFoundError(f"File {self._filename} not found.") dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._source = dfs self._n_items = safe_length(dfs.ItemInfo) self._n_timesteps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items, item_numbers, _ = get_valid_items_and_timesteps(self, items, time_steps) dfs.Close() ds = self.__read(self._filename) ds = ds[item_numbers] if time_steps: ds = ds.isel(time_steps, axis=0) return ds
def scale(infilename, outfilename, offset=0.0, factor=1.0): """Apply scaling to any dfs file Parameters ---------- infilename: str full path to the input file outfilename: str full path to the output file offset: float, optional value to add to all items, default 0.0 factor: float, optional value to multiply to all items, default 1.0 """ copyfile(infilename, outfilename) dfs = DfsFileFactory.DfsGenericOpenEdit(outfilename) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs.ItemInfo) for timestep in range(n_time_steps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item + 1, timestep) d = to_numpy(itemdata.Data) time = itemdata.Time outdata = d * factor + offset darray = to_dotnet_float_array(outdata) dfs.WriteItemTimeStep(item + 1, timestep, time, darray)
def write(self, filename, data): """ Function: write to a pre-created dfsu file. filename: full path and filename to existing dfsu file data: list of matrices. len(data) must equal the number of items in the dfsu. Each matrix must be of dimension time,elements usage: write(filename, data) where data(nt,elements) Returns: Nothing """ # Open the dfs file for writing dfs = DfsFileFactory.DfsGenericOpenEdit(filename) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs.ItemInfo) deletevalue = dfs.FileInfo.DeleteValueFloat for i in range(n_time_steps): for item in range(n_items): d = data[item][i, :] d[np.isnan(d)] = deletevalue darray = Array[System.Single](np.array(d.reshape(d.size, 1)[:, 0])) dfs.WriteItemTimeStepNext(0, darray) dfs.Close()
def write(self, filename, data): """ Function: write to a pre-created dfs2 file. filename: full path and filename to existing dfs2 file data: list of matrices. len(data) must equal the number of items in the dfs2. Easch matrix must be of dimension y,x,time usage: write( filename, data) where data( y, x, nt) Returns: Nothing """ # Open the dfs file for writing dfs = DfsFileFactory.Dfs2FileOpenEdit(filename) # Determine the size of the grid number_y = dfs.SpatialAxis.YCount number_x = dfs.SpatialAxis.XCount n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs.ItemInfo) deletevalue = -1e-035 if not all(np.shape(d)[0] == n_time_steps for d in data): raise Warning( "ERROR data matrices in the time dimension do not all match in the data list. " "Data is list of matices [time,y,x]" ) if not all(np.shape(d)[1] == number_y for d in data): raise Warning( "ERROR data matrices in the Y dimension do not all match in the data list. " "Data is list of matices [time,y,x]" ) if not all(np.shape(d)[2] == number_x for d in data): raise Warning( "ERROR data matrices in the X dimension do not all match in the data list. " "Data is list of matices [time, y, x]" ) if not len(data) == n_items: raise Warning( "The number of matrices in data do not match the number of items in the dfs2 file." ) for it in range(n_time_steps): for item in range(n_items): d = data[item][it, :, :] d[np.isnan(d)] = deletevalue d = d.reshape(number_y, number_x) d = np.flipud(d) darray = to_dotnet_float_array(d.reshape(d.size, 1)[:, 0]) dfs.WriteItemTimeStepNext(0, darray) dfs.Close()
def grid_coordinates(self, dfs3file): """Function: Returns the Grid information Usage: [X0, Y0, dx, dy, nx, ny, nz, nt] = grid_coordinates( filename ) dfs3file a full path and filename to the dfs3 file Returns: X0, Y0: bottom left coordinates dx, dy: grid size in x and y directions nx, ny, nz: number of grid elements in the x, y and z direction nt: number of time steps """ dfs = DfsFileFactory.DfsGenericOpen(dfs3file) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis dx = axis.Dx dy = axis.Dy x0 = axis.X0 y0 = axis.Y0 yNum = axis.YCount xNum = axis.XCount zNum = axis.ZCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps dfs.Close() return x0, y0, dx, dy, xNum, yNum, zNum, nt
def diff(infilename_a: str, infilename_b: str, outfilename: str) -> None: """Calculate difference between two dfs files (a-b) Parameters ---------- infilename_a: str full path to the first input file infilename_b: str full path to the second input file outfilename: str full path to the output file """ copyfile(infilename_a, outfilename) dfs_i_a = DfsFileFactory.DfsGenericOpen(infilename_a) dfs_i_b = DfsFileFactory.DfsGenericOpen(infilename_b) dfs_o = DfsFileFactory.DfsGenericOpenEdit(outfilename) deletevalue = dfs_i_a.FileInfo.DeleteValueFloat n_time_steps = dfs_i_a.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs_i_a.ItemInfo) # TODO Add checks to verify identical structure of file a and b for timestep in trange(n_time_steps): for item in range(n_items): itemdata_a = dfs_i_a.ReadItemTimeStep(item + 1, timestep) d_a = to_numpy(itemdata_a.Data) d_a[d_a == deletevalue] = np.nan itemdata_b = dfs_i_b.ReadItemTimeStep(item + 1, timestep) d_b = to_numpy(itemdata_b.Data) d_b[d_b == deletevalue] = np.nan time = itemdata_a.Time outdata = d_a - d_b darray = to_dotnet_float_array(outdata) dfs_o.WriteItemTimeStep(item + 1, timestep, time, darray) dfs_i_a.Close() dfs_i_b.Close() dfs_o.Close()
def _read_dfs1_header(self): if not os.path.isfile(self._filename): raise FileNotFoundError(self._filename) self._dfs = DfsFileFactory.Dfs1FileOpen(self._filename) self._dx = self._dfs.SpatialAxis.Dx self._nx = self._dfs.SpatialAxis.XCount self._read_header()
def read(self, items=None, time_steps=None): """ Read data from a dfs0 file. Parameters ---------- items: list[int] or list[str], optional Read only selected items, by number (0-based), or by name time_steps: str, int or list[int], optional Read only selected time_steps Returns ------- Dataset A dataset with data dimensions [t] """ if not os.path.exists(self._filename): raise FileNotFoundError(f"File {self._filename} not found.") dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._source = dfs self._n_items = safe_length(dfs.ItemInfo) item_numbers = _valid_item_numbers(dfs.ItemInfo, items) self._n_timesteps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if self._timeaxistype == TimeAxisType.NonEquidistantCalendar and isinstance( time_steps, str ): sel_time_step_str = time_steps time_steps = range(self._n_timesteps) else: sel_time_step_str = None dfs.Close() ds = self.__read(self._filename) ds = ds[item_numbers] if time_steps: ds = ds.isel(time_steps, axis=0) if sel_time_step_str: parts = sel_time_step_str.split(",") if len(parts) == 1: parts.append(parts[0]) # end=start if parts[0] == "": sel = slice(parts[1]) # stop only elif parts[1] == "": sel = slice(parts[0], None) # start only else: sel = slice(parts[0], parts[1]) ds = ds[sel] return ds
def _clone(infilename, outfilename): """Clone a dfs file Parameters ---------- infilename : str input filename outfilename : str output filename Returns ------- DfsFile MIKE generic dfs file object """ source = DfsFileFactory.DfsGenericOpen(infilename) fi = source.FileInfo builder = DfsBuilder.Create(fi.FileTitle, fi.ApplicationTitle, fi.ApplicationVersion) # Set up the header builder.SetDataType(fi.DataType) builder.SetGeographicalProjection(fi.Projection) builder.SetTemporalAxis(fi.TimeAxis) builder.SetItemStatisticsType(fi.StatsType) builder.DeleteValueByte = fi.DeleteValueByte builder.DeleteValueDouble = fi.DeleteValueDouble builder.DeleteValueFloat = fi.DeleteValueFloat builder.DeleteValueInt = fi.DeleteValueInt builder.DeleteValueUnsignedInt = fi.DeleteValueUnsignedInt # Copy custom blocks - if any for customBlock in fi.CustomBlocks: builder.AddCustomBlock(customBlock) # Copy dynamic items for itemInfo in source.ItemInfo: builder.AddDynamicItem(itemInfo) # Create file builder.CreateFile(outfilename) # Copy static items while True: static_item = source.ReadStaticItemNext() if static_item is None: break builder.AddStaticItem(static_item) # Get the file file = builder.GetFile() source.Close() return file
def scale(infilename, outfilename, offset=0.0, factor=1.0, item_numbers=None, item_names=None): """Apply scaling to any dfs file Parameters ---------- infilename: str full path to the input file outfilename: str full path to the output file offset: float, optional value to add to all items, default 0.0 factor: float, optional value to multiply to all items, default 1.0 item_numbers: list[int], optional Process only selected items, by number (0-based) item_names: list[str], optional Process only selected items, by name, takes precedence over item_numbers """ copyfile(infilename, outfilename) dfs = DfsFileFactory.DfsGenericOpenEdit(outfilename) if item_names is not None: item_numbers = find_item(dfs, item_names) if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps n_items = len(item_numbers) deletevalue = dfs.FileInfo.DeleteValueFloat for timestep in range(n_time_steps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, timestep) time = itemdata.Time d = to_numpy(itemdata.Data) d[d == deletevalue] = np.nan outdata = d * factor + offset outdata[np.isnan(outdata)] = deletevalue darray = to_dotnet_float_array(outdata) dfs.WriteItemTimeStep(item_numbers[item] + 1, timestep, time, darray) dfs.Close()
def write(self, filename, data): """Writes data to the pre-created dfs0 file. filename --> file path to existing dfs0 file. data --> numpy matrix with data. """ if not path.exists(filename): raise Warning("filename - File does not Exist %s", filename) try: dfs = DfsFileFactory.DfsGenericOpenEdit(filename) except IOError: print('cannot open', filename) delete_value = dfs.FileInfo.DeleteValueFloat n_items = len(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if len(np.shape(data)) == 1: data = data.reshape(len(data), 1) # Makes sure the data to write to the file matches the dfs0 file if nt != data.shape[0]: print("Inconsistent data size. nt (row count) must be size" + str(nt)) # quit() if n_items != data.shape[1]: print( "Inconsistent data size. number of items (column count) must be size" + str(n_items)) data[np.isnan(data)] = delete_value d = Array.CreateInstance(System.Single, 1) # Get the date times in seconds (from start) t = [] for i in range(nt): itemData = dfs.ReadItemTimeStep(1, i) newTime = DfsExtensions.TimeInSeconds(itemData, dfs.FileInfo.TimeAxis) t.append(newTime) dfs.Reset() # COPY OVER THE DATA for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): tit = System.Double(t[it]) for ii in range(len(dfs.ItemInfo)): d = Array[System.Single](np.array([[data[it, ii]]])) dfs.WriteItemTimeStepNext(tit, d) dfs.Close()
def _read_dfs2_header(self): if not os.path.isfile(self._filename): raise Exception(f"file {self._filename} does not exist!") self._dfs = DfsFileFactory.Dfs2FileOpen(self._filename) self._dx = self._dfs.SpatialAxis.Dx self._dy = self._dfs.SpatialAxis.Dy self._nx = self._dfs.SpatialAxis.XCount self._ny = self._dfs.SpatialAxis.YCount self._read_header()
def sum(infilename_a, infilename_b, outfilename): """Sum two dfs files (a+b) Parameters ---------- infilename_a: str full path to the first input file infilename_b: str full path to the second input file outfilename: str full path to the output file """ copyfile(infilename_a, outfilename) dfs_i_a = DfsFileFactory.DfsGenericOpen(infilename_a) dfs_i_b = DfsFileFactory.DfsGenericOpen(infilename_b) dfs_o = DfsFileFactory.DfsGenericOpenEdit(outfilename) n_time_steps = dfs_i_a.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs_i_a.ItemInfo) # TODO Add checks to verify identical structure of file a and b for timestep in range(n_time_steps): for item in range(n_items): itemdata_a = dfs_i_a.ReadItemTimeStep(item + 1, timestep) d_a = to_numpy(itemdata_a.Data) itemdata_b = dfs_i_b.ReadItemTimeStep(item + 1, timestep) d_b = to_numpy(itemdata_b.Data) time = itemdata_a.Time outdata = d_a + d_b darray = to_dotnet_float_array(outdata) dfs_o.WriteItemTimeStep(item + 1, timestep, time, darray) dfs_i_a.Close() dfs_i_b.Close() dfs_o.Close()
def _read_dfs2_header(self): if not os.path.isfile(self._filename): raise Exception(f"file {self._filename} does not exist!") self._dfs = DfsFileFactory.Dfs2FileOpen(self._filename) self._dx = self._dfs.SpatialAxis.Dx self._dy = self._dfs.SpatialAxis.Dy self._nx = self._dfs.SpatialAxis.XCount self._ny = self._dfs.SpatialAxis.YCount if self._dfs.FileInfo.TimeAxis.TimeAxisType == 4: self._is_equidistant = False self._read_header()
def scale( infilename: str, outfilename: str, offset: float = 0.0, factor: float = 1.0, items: Union[List[str], List[int]] = None, ) -> None: """Apply scaling to any dfs file Parameters ---------- infilename: str full path to the input file outfilename: str full path to the output file offset: float, optional value to add to all items, default 0.0 factor: float, optional value to multiply to all items, default 1.0 items: List[str] or List[int], optional Process only selected items, by number (0-based) """ copyfile(infilename, outfilename) dfs = DfsFileFactory.DfsGenericOpenEdit(outfilename) item_numbers = _valid_item_numbers(dfs.ItemInfo, items) n_items = len(item_numbers) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps deletevalue = dfs.FileInfo.DeleteValueFloat for timestep in trange(n_time_steps, disable=not show_progress): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, timestep) time = itemdata.Time d = to_numpy(itemdata.Data) d[d == deletevalue] = np.nan outdata = d * factor + offset outdata[np.isnan(outdata)] = deletevalue darray = to_dotnet_float_array(outdata) dfs.WriteItemTimeStep(item_numbers[item] + 1, timestep, time, darray) dfs.Close()
def __read(self, filename): """Read data from the dfs0 file """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) dfs = DfsFileFactory.DfsGenericOpen(filename) self._dfs = dfs n_items = safe_length(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items = [] for i in range(n_items): eumItem = dfs.ItemInfo[i].Quantity.Item eumUnit = dfs.ItemInfo[i].Quantity.Unit name = dfs.ItemInfo[i].Name itemtype = EUMType(eumItem) unit = EUMUnit(eumUnit) item = ItemInfo(name, itemtype, unit) items.append(item) # BULK READ THE DFS0 dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) t = [] starttime = dfs.FileInfo.TimeAxis.StartDateTime # EMPTY Data Block for copying the Results for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): t.append( starttime.AddSeconds( dfsdata[it, 0]).ToString("yyyy-MM-dd HH:mm:ss")) # Copies the System Array to a numpy matrix # First column in the time (the rest is the data) data = np.fromiter(dfsdata, np.float64).reshape(nt, n_items + 1)[:, 1::] mask = np.isclose(data, dfs.FileInfo.DeleteValueFloat, atol=1e-36) data[mask] = np.nan # mask = np.isclose( # data, dfs.FileInfo.DeleteValueDouble, atol=1e-34 # ) # TODO needs to be verified # data[mask] = np.nan dfs.Close() return data, t, items
def write(self, filename, data): """write overwrites an existing dfs0 file. Parameters ---------- filename: str Full path and filename to dfs0 to be modified. data: list[np.array] data to overwrite """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) try: dfs = DfsFileFactory.DfsGenericOpenEdit(filename) except IOError: print("cannot open", filename) delete_value = dfs.FileInfo.DeleteValueFloat n_items = len(dfs.ItemInfo) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps # Makes sure the data to write to the file matches the dfs0 file if n_time_steps != data[0].shape[0]: raise Exception( f"Inconsistent data size. nt (row count) must be size {n_time_steps}" ) if n_items != len(data): raise Exception(f"Number of items must be size {n_items}") for i in range(n_items): d = data[i] d[np.isnan(d)] = delete_value # Get the date times in seconds (from start) dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) t_seconds = [dfsdata[i,0] for i in range(n_time_steps)] dfs.Reset() data1 = np.stack(data, axis=1) Dfs0Util.WriteDfs0DataDouble(dfs, t_seconds, to_dotnet_array(data1)) dfs.Close()
def write(self, dfs3file, data): """ Function: write to a pre-created dfs3 file. Only ONE item supported. NOTE: The dfs3 file must be pre-created with corresponding y,x, z dimensions and number of time steps. The Data Matrix size ( y, x, z, nt) usage: write( filename, data) where data( y, x, z, nt) Returns: Nothing """ # Open the dfs file for writing dfs = DfsFileFactory.Dfs3FileOpenEdit(dfs3file) # Determine the size of the grid yNum = dfs.SpatialAxis.YCount xNum = dfs.SpatialAxis.XCount zNum = dfs.SpatialAxis.ZCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps deletevalue = dfs.FileInfo.DeleteValueFloat if data.shape[0] != yNum: sys.exit("ERROR Y dimension does not match") elif data.shape[1] != xNum: sys.exit("ERROR X dimension does not match") elif data.shape[2] != zNum: sys.exit("ERROR X dimension does not match") elif data.shape[3] != nt: sys.exit("ERROR Number of Time Steps dimension does not match") for it in range(nt): d = data[:, :, :, it] d[np.isnan(d)] = deletevalue d = d.swapaxes(1, 2).swapaxes(1, 0) d = d[:, ::-1, :] darray = Array[System.Single](np.asarray(d).reshape(-1)) dfs.WriteItemTimeStepNext(0, darray) dfs.Close()
def __read(self, filename): """Read data from the dfs0 file """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) dfs = DfsFileFactory.DfsGenericOpen(filename) self._dfs = dfs n_items = safe_length(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items = [] for i in range(n_items): eumItem = dfs.ItemInfo[i].Quantity.Item eumUnit = dfs.ItemInfo[i].Quantity.Unit name = dfs.ItemInfo[i].Name itemtype = EUMType(eumItem) unit = EUMUnit(eumUnit) item = ItemInfo(name, itemtype, unit) items.append(item) # BULK READ THE DFS0 dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) # First column is the time (the rest is the data) t_seconds = [dfsdata[i, 0] for i in range(nt)] start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] # TODO use to_numpy ? data = np.fromiter(dfsdata, np.float64).reshape(nt, n_items + 1)[:, 1::] mask = np.isclose(data, dfs.FileInfo.DeleteValueFloat, atol=1e-36) data[mask] = np.nan # mask = np.isclose( # data, dfs.FileInfo.DeleteValueDouble, atol=1e-34 # ) # TODO needs to be verified # data[mask] = np.nan dfs.Close() return data, time, items
def __read(self, filename): """ Read data from a dfs0 file. """ if not os.path.exists(filename): raise FileNotFoundError(f"File {filename} not found.") self._dfs = DfsFileFactory.DfsGenericOpen(filename) self._n_items = safe_length(self._dfs.ItemInfo) self._n_timesteps = self._dfs.FileInfo.TimeAxis.NumberOfTimeSteps self._time_column_index = 0 # First column is time (the rest is data) raw_data = Dfs0Util.ReadDfs0DataDouble(self._dfs) # Bulk read the data data = self.__to_numpy_with_nans(raw_data) time = list(self.__get_time(raw_data)) items = list(self.__get_items()) self._dfs.Close() return data, time, items
def __read(self, filename): """ Read all data from a dfs0 file. """ self._time_column_index = 0 # First column is time (the rest is data) self._dfs = DfsFileFactory.DfsGenericOpen(filename) raw_data = Dfs0Util.ReadDfs0DataDouble(self._dfs) # Bulk read the data matrix = self.__to_numpy_with_nans(raw_data) data = [] for i in range(matrix.shape[1]): data.append(matrix[:, i]) time = list(self.__get_time(raw_data)) items = list(self.__get_items()) self._dfs.Close() return Dataset(data, time, items)
def __read(self, filename): """Read data from the dfs0 file """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) dfs = DfsFileFactory.DfsGenericOpen(filename) n_items = safe_length(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps names = [] for i in range(n_items): names.append(dfs.ItemInfo[i].Name) # BULK READ THE DFS0 dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) t = [] starttime = dfs.FileInfo.TimeAxis.StartDateTime # EMPTY Data Block for copying the Results for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): t.append( starttime.AddSeconds( dfsdata[it, 0]).ToString("yyyy-MM-dd HH:mm:ss")) # Copies the System Array to a numpy matrix # First column in the time (the rest is the data) data = np.fromiter(dfsdata, np.float64).reshape(nt, n_items + 1)[:, 1::] data[data == -1.0000000180025095e-35] = np.nan data[data == -1.0000000031710769e-30] = np.nan data[data == dfs.FileInfo.DeleteValueFloat] = np.nan data[data == dfs.FileInfo.DeleteValueDouble] = np.nan dfs.Close() return data, t, names
def write(self, filename, data): """Overwrite a pre-created dfsu file. Parameters ---------- filename: str full path and filename to existing dfsu file data: list[np.array] list of matrices. len(data) must equal the number of items in the dfsu. Each matrix must be of dimension time,elements """ # Open the dfs file for writing dfs = DfsFileFactory.DfsGenericOpenEdit(filename) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs.ItemInfo) if len(data) != n_items: dfs.Close() raise ValueError( "Number of items in data must equal number of items in the file" ) if data[0].shape[0] != n_time_steps: dfs.Close() raise ValueError( "Number of timesteps in data must equal number of timesteps in the file" ) deletevalue = dfs.FileInfo.DeleteValueFloat for i in range(n_time_steps): for item in range(n_items): d = data[item][i, :] d[np.isnan(d)] = deletevalue darray = to_dotnet_float_array(d) dfs.WriteItemTimeStepNext(0, darray) dfs.Close()
def find_nearest_element( self, lon, lat, ): """Find index of closest element Parameters ---------- lon: float longitude lat: float latitude Returns ------- (int,int): indexes in y, x """ dfs = DfsFileFactory.Dfs2FileOpen(self._filename) self._dfs = dfs projection = self._dfs.FileInfo.Projection axis = self._dfs.SpatialAxis cart = Cartography( projection.WKTString, projection.Longitude, projection.Latitude, projection.Orientation, ) # C# out parameters must be handled in special way (_, xx, yy) = cart.Geo2Xy(lon, lat, 0.0, 0.0) j = int(xx / axis.Dx + 0.5) k = axis.YCount - int(yy / axis.Dy + 0.5) - 1 j = min(max(0, j), axis.XCount - 1) k = min(max(0, k), axis.YCount - 1) return k, j
def _validate_and_open_dfs(filename, data): if not os.path.exists(filename): raise FileNotFoundError(filename) try: dfs = DfsFileFactory.DfsGenericOpenEdit(filename) except IOError: raise IOError(f"Cannot open {filename}.") n_items = len(dfs.ItemInfo) n_time_steps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps # Match the data to write to the existing dfs0 file if n_time_steps != data[0].shape[0]: raise ValueError( f"Inconsistent data size. Number of time steps (row count) is {data[0].shape[0]}. Expected {n_time_steps}." ) if n_items != len(data): raise ValueError(f"The number of items is {len(data)}. Expected {n_items}.") return dfs, n_items, n_time_steps
def _read_header(self): if not os.path.exists(self._filename): raise FileNotFoundError(self._filename) dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._deletevalue = dfs.FileInfo.DeleteValueFloat # Read items self._n_items = safe_length(dfs.ItemInfo) self._items = _get_item_info(dfs.ItemInfo, list(range(self._n_items))) self._timeaxistype = TimeAxisType(dfs.FileInfo.TimeAxis.TimeAxisType) if self._timeaxistype in [ TimeAxisType.EquidistantCalendar, TimeAxisType.NonEquidistantCalendar, ]: self._start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) else: # relative time axis self._start_time = datetime(1970, 1, 1) dfs.Close()
def read(self, filename, item_numbers=None, item_names=None): """Read data from the dfs1 file Usage: read(filename, item_numbers=None, item_names=None) filename full path to the dfs1 file. item_numbers read only the item_numbers in the array specified (0 base) item_names read only the items in the array specified, (takes precedence over item_numbers) Return: Dataset(data, time, items) where data[nt,y,x] """ # NOTE. Item numbers are base 0 (everything else in the dfs is base 0) # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(filename) if item_names is not None: item_numbers = find_item(dfs, item_names) if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis yNum = axis.YCount xNum = axis.XCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if nt == 0: raise Warning( "Static files (with no dynamic items) are not supported.") nt = 1 deleteValue = dfs.FileInfo.DeleteValueFloat n_items = len(item_numbers) data_list = [] for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(nt, yNum, xNum), dtype=float) data_list.append(data) t = [] startTime = dfs.FileInfo.TimeAxis.StartDateTime for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) d = d.reshape(yNum, xNum) d = np.flipud(d) d[d == deleteValue] = np.nan data_list[item][it, :, :] = d t.append( startTime.AddSeconds( itemdata.Time).ToString("yyyy-MM-dd HH:mm:ss")) time = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in t] items = get_item_info(dfs, item_numbers) dfs.Close() return Dataset(data_list, time, items)
def concat(infilenames, outfilename): """Concatenates files along the time axis If files are overlapping, the last one will be used. Parameters ---------- infilenames: list of str filenames to concatenate outfilename: str filename Notes ------ The list of input files have to be sorted, i.e. in chronological order """ dfs_i_a = DfsFileFactory.DfsGenericOpen(infilenames[0]) dfs_o = _clone(infilenames[0], outfilename) n_items = safe_length(dfs_i_a.ItemInfo) for i, infilename in enumerate(infilenames): dfs_i = DfsFileFactory.DfsGenericOpen(infilename) n_time_steps = dfs_i.FileInfo.TimeAxis.NumberOfTimeSteps dt = dfs_i.FileInfo.TimeAxis.TimeStep start_time = from_dotnet_datetime(dfs_i.FileInfo.TimeAxis.StartDateTime) if i > 0 and start_time > current_time + timedelta(seconds=dt): dfs_o.Close() os.remove(outfilename) raise Exception("Gap in time axis detected - not supported") current_time = start_time if i < (len(infilenames) - 1): dfs_n = DfsFileFactory.DfsGenericOpen(infilenames[i + 1]) nf = dfs_n.FileInfo.TimeAxis.StartDateTime next_start_time = datetime( nf.Year, nf.Month, nf.Day, nf.Hour, nf.Minute, nf.Second ) for timestep in range(n_time_steps): current_time = start_time + timedelta(seconds=timestep * dt) if i < (len(infilenames) - 1): if current_time >= next_start_time: break for item in range(n_items): itemdata = dfs_i.ReadItemTimeStep(item + 1, timestep) d = to_numpy(itemdata.Data) darray = to_dotnet_float_array(d) dfs_o.WriteItemTimeStepNext(0, darray) dfs_o.Close()
def read_slice( self, dfs3file, lower_left_xy, upper_right_xy, items=None, layers=None, conservative=True, ): """Function: Read data from a dfs3 file within the locations chosen Usage: [data,time,name] = read( filename, lower_left_xy, upper_right_xy, items, conservative) dfs3file a full path and filename to the dfs3 file lower_left_xy list or array of size two with the X and the Y coordinate (same projection as the dfs3) upper_right_xy list or array of size two with the X and the Y coordinate (same projection as the dfs3) items list of indices (base 0) to read from layers list of layers to read conservative Default is true. Only include the grids within the given bounds (don't return those grids on the boarder) Returns 1) the data contained in a dfs3 file in a list of numpy matrices 2) time index 3) name of the items NOTE Returns data ( y, x, z, nt) 1) If coordinates is selected, then only return data at those coordinates 2) coordinates specified overules layers. 3) layer counts from the bottom """ data = self.read(dfs3file, items=items, layers=layers) dfs = DfsFileFactory.DfsGenericOpen(dfs3file) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis dx = axis.Dx dy = axis.Dy x0 = axis.X0 y0 = axis.Y0 yNum = axis.YCount xNum = axis.XCount top_left_y = y0 + (yNum + 1) * dy dfs.Close() # SLICE all the Data lower_left_x_index = (lower_left_xy[0] - x0) / dx lower_left_y_index = (top_left_y - lower_left_xy[1]) / dy upper_right_x_index = (upper_right_xy[0] - x0) / dx upper_right_y_index = (top_left_y - upper_right_xy[1]) / dy if conservative: lower_left_x_index = int(np.ceil(lower_left_x_index)) upper_right_x_index = int(np.floor(upper_right_x_index)) lower_left_y_index = int(np.floor(lower_left_y_index)) upper_right_y_index = int(np.ceil(upper_right_y_index)) else: lower_left_x_index = int(np.floor(lower_left_x_index)) upper_right_x_index = int(np.ceil(upper_right_x_index)) lower_left_y_index = int(np.ceil(lower_left_y_index)) upper_right_y_index = int(np.floor(upper_right_y_index)) if lower_left_x_index < 0: raise IndexError("lower_left_x_index < 0.") lower_left_x_index = 0 if upper_right_y_index < 0: raise IndexError("upper_right_y_index < 0.") upper_right_y_index = 0 if lower_left_y_index > yNum - 1: raise IndexError("lower_left_y_index > yNum - 1") lower_left_y_index = yNum - 1 if upper_right_x_index > xNum - 1: raise IndexError("upper_right_x_index > xNum - 1") upper_right_x_index = xNum - 1 for i in range(len(data[0])): data[0][i] = data[0][i][ upper_right_y_index:lower_left_y_index, lower_left_x_index:upper_right_x_index, :, :, ] return data
def read(self, items=None, layers=None, coordinates=None, time_steps=None): """Function: Read data from a dfs3 file Usage: [data,time,name] = read( filename, items, layers=None, coordinates=None) items list of indices (base 0) to read from. If None then all the items. layers list of layer indices (base 0) to read coordinates list of list (x,y,layer) integers ( 0,0 at Bottom Left of Grid !! ) example coordinates = [[2,5,1], [11,41,2]] Returns 1) the data contained in a dfs3 file in a list of numpy matrices 2) time index 3) name of the items NOTE Returns Dataset with data (t, z, y, x) 1) If coordinates is selected, then only return data at those coordinates 2) coordinates specified overules layers. 3) layer counts from the bottom """ # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(self._filename) item_numbers = _valid_item_numbers(dfs.ItemInfo, items) n_items = len(item_numbers) time_steps = _valid_timesteps(dfs.FileInfo, time_steps) nt = len(time_steps) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis zNum = axis.ZCount yNum = axis.YCount xNum = axis.XCount deleteValue = dfs.FileInfo.DeleteValueFloat data_list = [] if coordinates is None: # if nt is 0, then the dfs is 'static' and must be handled differently if nt != 0: for item in range(n_items): if layers is None: # Initialize an empty data block data = np.ndarray(shape=(nt, zNum, yNum, xNum), dtype=float) data_list.append(data) else: data = np.ndarray(shape=(nt, len(layers), yNum, xNum), dtype=float) data_list.append(data) else: raise ValueError( "Static dfs3 files (with no time steps) are not supported." ) else: ncoordinates = len(coordinates) for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(nt, ncoordinates), dtype=float) data_list.append(data) t_seconds = np.zeros(nt, dtype=float) startTime = dfs.FileInfo.TimeAxis.StartDateTime if coordinates is None: for it_number, it in enumerate(time_steps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) # DO a direct copy instead of eleement by elment d = d.reshape(zNum, yNum, xNum) # .swapaxes(0, 2).swapaxes(0, 1) d = np.flipud(d) d[d == deleteValue] = np.nan if layers is None: data_list[item][it_number, :, :, :] = d else: for l in range(len(layers)): data_list[item][it_number, l, :, :] = d[layers[l], :, :] t_seconds[it_number] = itemdata.Time else: indices = [ self.__calculate_index(xNum, yNum, zNum, x, y, z) for x, y, z in coordinates ] for it in range(nt): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) d = np.array([itemdata.Data[i] for i in indices]) d[d == deleteValue] = np.nan data_list[item][it, :] = d t_seconds[it] = itemdata.Time start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] items = _get_item_info(dfs.ItemInfo, item_numbers) dfs.Close() return Dataset(data_list, time, items)