def read(self, items=None, time_steps=None): """ Read data from a dfs0 file. Parameters ---------- items: list[int] or list[str], optional Read only selected items, by number (0-based), or by name time_steps: int or list[int], optional Read only selected time_steps Returns ------- Dataset A dataset with data dimensions [t] """ if not os.path.exists(self._filename): raise FileNotFoundError(f"File {self._filename} not found.") dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._source = dfs self._n_items = safe_length(dfs.ItemInfo) self._n_timesteps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items, item_numbers, _ = get_valid_items_and_timesteps(self, items, time_steps) dfs.Close() ds = self.__read(self._filename) ds = ds[item_numbers] if time_steps: ds = ds.isel(time_steps, axis=0) return ds
def grid_coordinates(self, dfs3file): """Function: Returns the Grid information Usage: [X0, Y0, dx, dy, nx, ny, nz, nt] = grid_coordinates( filename ) dfs3file a full path and filename to the dfs3 file Returns: X0, Y0: bottom left coordinates dx, dy: grid size in x and y directions nx, ny, nz: number of grid elements in the x, y and z direction nt: number of time steps """ dfs = DfsFileFactory.DfsGenericOpen(dfs3file) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis dx = axis.Dx dy = axis.Dy x0 = axis.X0 y0 = axis.Y0 yNum = axis.YCount xNum = axis.XCount zNum = axis.ZCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps dfs.Close() return x0, y0, dx, dy, xNum, yNum, zNum, nt
def diff(infilename_a: str, infilename_b: str, outfilename: str) -> None: """Calculate difference between two dfs files (a-b) Parameters ---------- infilename_a: str full path to the first input file infilename_b: str full path to the second input file outfilename: str full path to the output file """ copyfile(infilename_a, outfilename) dfs_i_a = DfsFileFactory.DfsGenericOpen(infilename_a) dfs_i_b = DfsFileFactory.DfsGenericOpen(infilename_b) dfs_o = DfsFileFactory.DfsGenericOpenEdit(outfilename) deletevalue = dfs_i_a.FileInfo.DeleteValueFloat n_time_steps = dfs_i_a.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs_i_a.ItemInfo) # TODO Add checks to verify identical structure of file a and b for timestep in trange(n_time_steps): for item in range(n_items): itemdata_a = dfs_i_a.ReadItemTimeStep(item + 1, timestep) d_a = to_numpy(itemdata_a.Data) d_a[d_a == deletevalue] = np.nan itemdata_b = dfs_i_b.ReadItemTimeStep(item + 1, timestep) d_b = to_numpy(itemdata_b.Data) d_b[d_b == deletevalue] = np.nan time = itemdata_a.Time outdata = d_a - d_b darray = to_dotnet_float_array(outdata) dfs_o.WriteItemTimeStep(item + 1, timestep, time, darray) dfs_i_a.Close() dfs_i_b.Close() dfs_o.Close()
def read(self, items=None, time_steps=None): """ Read data from a dfs0 file. Parameters ---------- items: list[int] or list[str], optional Read only selected items, by number (0-based), or by name time_steps: str, int or list[int], optional Read only selected time_steps Returns ------- Dataset A dataset with data dimensions [t] """ if not os.path.exists(self._filename): raise FileNotFoundError(f"File {self._filename} not found.") dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._source = dfs self._n_items = safe_length(dfs.ItemInfo) item_numbers = _valid_item_numbers(dfs.ItemInfo, items) self._n_timesteps = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if self._timeaxistype == TimeAxisType.NonEquidistantCalendar and isinstance( time_steps, str ): sel_time_step_str = time_steps time_steps = range(self._n_timesteps) else: sel_time_step_str = None dfs.Close() ds = self.__read(self._filename) ds = ds[item_numbers] if time_steps: ds = ds.isel(time_steps, axis=0) if sel_time_step_str: parts = sel_time_step_str.split(",") if len(parts) == 1: parts.append(parts[0]) # end=start if parts[0] == "": sel = slice(parts[1]) # stop only elif parts[1] == "": sel = slice(parts[0], None) # start only else: sel = slice(parts[0], parts[1]) ds = ds[sel] return ds
def _clone(infilename, outfilename): """Clone a dfs file Parameters ---------- infilename : str input filename outfilename : str output filename Returns ------- DfsFile MIKE generic dfs file object """ source = DfsFileFactory.DfsGenericOpen(infilename) fi = source.FileInfo builder = DfsBuilder.Create(fi.FileTitle, fi.ApplicationTitle, fi.ApplicationVersion) # Set up the header builder.SetDataType(fi.DataType) builder.SetGeographicalProjection(fi.Projection) builder.SetTemporalAxis(fi.TimeAxis) builder.SetItemStatisticsType(fi.StatsType) builder.DeleteValueByte = fi.DeleteValueByte builder.DeleteValueDouble = fi.DeleteValueDouble builder.DeleteValueFloat = fi.DeleteValueFloat builder.DeleteValueInt = fi.DeleteValueInt builder.DeleteValueUnsignedInt = fi.DeleteValueUnsignedInt # Copy custom blocks - if any for customBlock in fi.CustomBlocks: builder.AddCustomBlock(customBlock) # Copy dynamic items for itemInfo in source.ItemInfo: builder.AddDynamicItem(itemInfo) # Create file builder.CreateFile(outfilename) # Copy static items while True: static_item = source.ReadStaticItemNext() if static_item is None: break builder.AddStaticItem(static_item) # Get the file file = builder.GetFile() source.Close() return file
def sum(infilename_a, infilename_b, outfilename): """Sum two dfs files (a+b) Parameters ---------- infilename_a: str full path to the first input file infilename_b: str full path to the second input file outfilename: str full path to the output file """ copyfile(infilename_a, outfilename) dfs_i_a = DfsFileFactory.DfsGenericOpen(infilename_a) dfs_i_b = DfsFileFactory.DfsGenericOpen(infilename_b) dfs_o = DfsFileFactory.DfsGenericOpenEdit(outfilename) n_time_steps = dfs_i_a.FileInfo.TimeAxis.NumberOfTimeSteps n_items = safe_length(dfs_i_a.ItemInfo) # TODO Add checks to verify identical structure of file a and b for timestep in range(n_time_steps): for item in range(n_items): itemdata_a = dfs_i_a.ReadItemTimeStep(item + 1, timestep) d_a = to_numpy(itemdata_a.Data) itemdata_b = dfs_i_b.ReadItemTimeStep(item + 1, timestep) d_b = to_numpy(itemdata_b.Data) time = itemdata_a.Time outdata = d_a + d_b darray = to_dotnet_float_array(outdata) dfs_o.WriteItemTimeStep(item + 1, timestep, time, darray) dfs_i_a.Close() dfs_i_b.Close() dfs_o.Close()
def __read(self, filename): """Read data from the dfs0 file """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) dfs = DfsFileFactory.DfsGenericOpen(filename) self._dfs = dfs n_items = safe_length(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items = [] for i in range(n_items): eumItem = dfs.ItemInfo[i].Quantity.Item eumUnit = dfs.ItemInfo[i].Quantity.Unit name = dfs.ItemInfo[i].Name itemtype = EUMType(eumItem) unit = EUMUnit(eumUnit) item = ItemInfo(name, itemtype, unit) items.append(item) # BULK READ THE DFS0 dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) t = [] starttime = dfs.FileInfo.TimeAxis.StartDateTime # EMPTY Data Block for copying the Results for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): t.append( starttime.AddSeconds( dfsdata[it, 0]).ToString("yyyy-MM-dd HH:mm:ss")) # Copies the System Array to a numpy matrix # First column in the time (the rest is the data) data = np.fromiter(dfsdata, np.float64).reshape(nt, n_items + 1)[:, 1::] mask = np.isclose(data, dfs.FileInfo.DeleteValueFloat, atol=1e-36) data[mask] = np.nan # mask = np.isclose( # data, dfs.FileInfo.DeleteValueDouble, atol=1e-34 # ) # TODO needs to be verified # data[mask] = np.nan dfs.Close() return data, t, items
def __read(self, filename): """Read data from the dfs0 file """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) dfs = DfsFileFactory.DfsGenericOpen(filename) self._dfs = dfs n_items = safe_length(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items = [] for i in range(n_items): eumItem = dfs.ItemInfo[i].Quantity.Item eumUnit = dfs.ItemInfo[i].Quantity.Unit name = dfs.ItemInfo[i].Name itemtype = EUMType(eumItem) unit = EUMUnit(eumUnit) item = ItemInfo(name, itemtype, unit) items.append(item) # BULK READ THE DFS0 dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) # First column is the time (the rest is the data) t_seconds = [dfsdata[i, 0] for i in range(nt)] start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] # TODO use to_numpy ? data = np.fromiter(dfsdata, np.float64).reshape(nt, n_items + 1)[:, 1::] mask = np.isclose(data, dfs.FileInfo.DeleteValueFloat, atol=1e-36) data[mask] = np.nan # mask = np.isclose( # data, dfs.FileInfo.DeleteValueDouble, atol=1e-34 # ) # TODO needs to be verified # data[mask] = np.nan dfs.Close() return data, time, items
def __read(self, filename): """ Read data from a dfs0 file. """ if not os.path.exists(filename): raise FileNotFoundError(f"File {filename} not found.") self._dfs = DfsFileFactory.DfsGenericOpen(filename) self._n_items = safe_length(self._dfs.ItemInfo) self._n_timesteps = self._dfs.FileInfo.TimeAxis.NumberOfTimeSteps self._time_column_index = 0 # First column is time (the rest is data) raw_data = Dfs0Util.ReadDfs0DataDouble(self._dfs) # Bulk read the data data = self.__to_numpy_with_nans(raw_data) time = list(self.__get_time(raw_data)) items = list(self.__get_items()) self._dfs.Close() return data, time, items
def __read(self, filename): """ Read all data from a dfs0 file. """ self._time_column_index = 0 # First column is time (the rest is data) self._dfs = DfsFileFactory.DfsGenericOpen(filename) raw_data = Dfs0Util.ReadDfs0DataDouble(self._dfs) # Bulk read the data matrix = self.__to_numpy_with_nans(raw_data) data = [] for i in range(matrix.shape[1]): data.append(matrix[:, i]) time = list(self.__get_time(raw_data)) items = list(self.__get_items()) self._dfs.Close() return Dataset(data, time, items)
def __read(self, filename): """Read data from the dfs0 file """ if not os.path.exists(filename): raise Warning("filename - File does not Exist %s", filename) dfs = DfsFileFactory.DfsGenericOpen(filename) n_items = safe_length(dfs.ItemInfo) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps names = [] for i in range(n_items): names.append(dfs.ItemInfo[i].Name) # BULK READ THE DFS0 dfsdata = Dfs0Util.ReadDfs0DataDouble(dfs) t = [] starttime = dfs.FileInfo.TimeAxis.StartDateTime # EMPTY Data Block for copying the Results for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): t.append( starttime.AddSeconds( dfsdata[it, 0]).ToString("yyyy-MM-dd HH:mm:ss")) # Copies the System Array to a numpy matrix # First column in the time (the rest is the data) data = np.fromiter(dfsdata, np.float64).reshape(nt, n_items + 1)[:, 1::] data[data == -1.0000000180025095e-35] = np.nan data[data == -1.0000000031710769e-30] = np.nan data[data == dfs.FileInfo.DeleteValueFloat] = np.nan data[data == dfs.FileInfo.DeleteValueDouble] = np.nan dfs.Close() return data, t, names
def _read_header(self): if not os.path.exists(self._filename): raise FileNotFoundError(self._filename) dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._deletevalue = dfs.FileInfo.DeleteValueFloat # Read items self._n_items = safe_length(dfs.ItemInfo) self._items = _get_item_info(dfs.ItemInfo, list(range(self._n_items))) self._timeaxistype = TimeAxisType(dfs.FileInfo.TimeAxis.TimeAxisType) if self._timeaxistype in [ TimeAxisType.EquidistantCalendar, TimeAxisType.NonEquidistantCalendar, ]: self._start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) else: # relative time axis self._start_time = datetime(1970, 1, 1) dfs.Close()
def _clone(infilename: str, outfilename: str, start_time=None, items=None) -> DfsFile: """Clone a dfs file Parameters ---------- infilename : str input filename outfilename : str output filename start_time : datetime, optional new start time for the new file, default items : list(int), optional clone only these items, default: all items Returns ------- DfsFile MIKE generic dfs file object """ source = DfsFileFactory.DfsGenericOpen(infilename) fi = source.FileInfo builder = DfsBuilder.Create(fi.FileTitle, fi.ApplicationTitle, fi.ApplicationVersion) # Set up the header builder.SetDataType(fi.DataType) builder.SetGeographicalProjection(fi.Projection) # Copy time axis time_axis = fi.TimeAxis if start_time is not None: dt = to_dotnet_datetime(start_time) time_axis.set_StartDateTime(dt) builder.SetTemporalAxis(time_axis) builder.SetItemStatisticsType(fi.StatsType) builder.DeleteValueByte = fi.DeleteValueByte builder.DeleteValueDouble = fi.DeleteValueDouble builder.DeleteValueFloat = fi.DeleteValueFloat builder.DeleteValueInt = fi.DeleteValueInt builder.DeleteValueUnsignedInt = fi.DeleteValueUnsignedInt # Copy custom blocks - if any for customBlock in fi.CustomBlocks: builder.AddCustomBlock(customBlock) # Copy dynamic items item_numbers = _valid_item_numbers(source.ItemInfo, items) for item in item_numbers: builder.AddDynamicItem(source.ItemInfo[item]) # Create file builder.CreateFile(outfilename) # Copy static items while True: static_item = source.ReadStaticItemNext() if static_item is None: break builder.AddStaticItem(static_item) # Get the file file = builder.GetFile() source.Close() return file
def read(self, filename, item_numbers=None, item_names=None, time_steps=None): """ Parameters --------- filename: str dfs2 filename item_numbers: list[int], optional Read only selected items, by number (0-based) item_names: list[str], optional Read only selected items, by name, takes precedence over item_numbers time_steps: list[int], optional Read only selected time_steps Returns ------- Dataset A dataset with data dimensions [t,y,x] """ # NOTE. Item numbers are base 0 (everything else in the dfs is base 0) # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(filename) if item_names is not None: item_numbers = find_item(dfs, item_names) if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if time_steps is None: time_steps = list(range(nt)) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis yNum = axis.YCount xNum = axis.XCount if nt == 0: raise Warning("Static files (with no dynamic items) are not supported.") nt = 1 for t in time_steps: if t > (nt - 1): raise ValueError(f"Trying to read timestep {t}: max timestep is {nt-1}") deleteValue = dfs.FileInfo.DeleteValueFloat n_items = len(item_numbers) data_list = [] for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(len(time_steps), yNum, xNum), dtype=float) data_list.append(data) t_seconds = np.zeros(len(time_steps), dtype=float) startTime = dfs.FileInfo.TimeAxis.StartDateTime for i in range(len(time_steps)): it = time_steps[i] for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) d = d.reshape(yNum, xNum) d = np.flipud(d) d[d == deleteValue] = np.nan data_list[item][i, :, :] = d t_seconds[i] = itemdata.Time start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] items = get_item_info(dfs, item_numbers) dfs.Close() return Dataset(data_list, time, items)
def read_slice( self, dfs3file, lower_left_xy, upper_right_xy, items=None, layers=None, conservative=True, ): """Function: Read data from a dfs3 file within the locations chosen Usage: [data,time,name] = read( filename, lower_left_xy, upper_right_xy, items, conservative) dfs3file a full path and filename to the dfs3 file lower_left_xy list or array of size two with the X and the Y coordinate (same projection as the dfs3) upper_right_xy list or array of size two with the X and the Y coordinate (same projection as the dfs3) items list of indices (base 0) to read from layers list of layers to read conservative Default is true. Only include the grids within the given bounds (don't return those grids on the boarder) Returns 1) the data contained in a dfs3 file in a list of numpy matrices 2) time index 3) name of the items NOTE Returns data ( y, x, z, nt) 1) If coordinates is selected, then only return data at those coordinates 2) coordinates specified overules layers. 3) layer counts from the bottom """ data = self.read(dfs3file, items=items, layers=layers) dfs = DfsFileFactory.DfsGenericOpen(dfs3file) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis dx = axis.Dx dy = axis.Dy x0 = axis.X0 y0 = axis.Y0 yNum = axis.YCount xNum = axis.XCount top_left_y = y0 + (yNum + 1) * dy dfs.Close() # SLICE all the Data lower_left_x_index = (lower_left_xy[0] - x0) / dx lower_left_y_index = (top_left_y - lower_left_xy[1]) / dy upper_right_x_index = (upper_right_xy[0] - x0) / dx upper_right_y_index = (top_left_y - upper_right_xy[1]) / dy if conservative: lower_left_x_index = int(np.ceil(lower_left_x_index)) upper_right_x_index = int(np.floor(upper_right_x_index)) lower_left_y_index = int(np.floor(lower_left_y_index)) upper_right_y_index = int(np.ceil(upper_right_y_index)) else: lower_left_x_index = int(np.floor(lower_left_x_index)) upper_right_x_index = int(np.ceil(upper_right_x_index)) lower_left_y_index = int(np.ceil(lower_left_y_index)) upper_right_y_index = int(np.floor(upper_right_y_index)) if lower_left_x_index < 0: raise IndexError("lower_left_x_index < 0.") lower_left_x_index = 0 if upper_right_y_index < 0: raise IndexError("upper_right_y_index < 0.") upper_right_y_index = 0 if lower_left_y_index > yNum - 1: raise IndexError("lower_left_y_index > yNum - 1") lower_left_y_index = yNum - 1 if upper_right_x_index > xNum - 1: raise IndexError("upper_right_x_index > xNum - 1") upper_right_x_index = xNum - 1 for i in range(len(data[0])): data[0][i] = data[0][i][ upper_right_y_index:lower_left_y_index, lower_left_x_index:upper_right_x_index, :, :, ] return data
def read(self, items=None, layers=None, coordinates=None, time_steps=None): """Function: Read data from a dfs3 file Usage: [data,time,name] = read( filename, items, layers=None, coordinates=None) items list of indices (base 0) to read from. If None then all the items. layers list of layer indices (base 0) to read coordinates list of list (x,y,layer) integers ( 0,0 at Bottom Left of Grid !! ) example coordinates = [[2,5,1], [11,41,2]] Returns 1) the data contained in a dfs3 file in a list of numpy matrices 2) time index 3) name of the items NOTE Returns Dataset with data (t, z, y, x) 1) If coordinates is selected, then only return data at those coordinates 2) coordinates specified overules layers. 3) layer counts from the bottom """ # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(self._filename) item_numbers = _valid_item_numbers(dfs.ItemInfo, items) n_items = len(item_numbers) time_steps = _valid_timesteps(dfs.FileInfo, time_steps) nt = len(time_steps) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis zNum = axis.ZCount yNum = axis.YCount xNum = axis.XCount deleteValue = dfs.FileInfo.DeleteValueFloat data_list = [] if coordinates is None: # if nt is 0, then the dfs is 'static' and must be handled differently if nt != 0: for item in range(n_items): if layers is None: # Initialize an empty data block data = np.ndarray(shape=(nt, zNum, yNum, xNum), dtype=float) data_list.append(data) else: data = np.ndarray(shape=(nt, len(layers), yNum, xNum), dtype=float) data_list.append(data) else: raise ValueError( "Static dfs3 files (with no time steps) are not supported." ) else: ncoordinates = len(coordinates) for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(nt, ncoordinates), dtype=float) data_list.append(data) t_seconds = np.zeros(nt, dtype=float) startTime = dfs.FileInfo.TimeAxis.StartDateTime if coordinates is None: for it_number, it in enumerate(time_steps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) # DO a direct copy instead of eleement by elment d = d.reshape(zNum, yNum, xNum) # .swapaxes(0, 2).swapaxes(0, 1) d = np.flipud(d) d[d == deleteValue] = np.nan if layers is None: data_list[item][it_number, :, :, :] = d else: for l in range(len(layers)): data_list[item][it_number, l, :, :] = d[layers[l], :, :] t_seconds[it_number] = itemdata.Time else: indices = [ self.__calculate_index(xNum, yNum, zNum, x, y, z) for x, y, z in coordinates ] for it in range(nt): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) d = np.array([itemdata.Data[i] for i in indices]) d[d == deleteValue] = np.nan data_list[item][it, :] = d t_seconds[it] = itemdata.Time start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] items = _get_item_info(dfs.ItemInfo, item_numbers) dfs.Close() return Dataset(data_list, time, items)
def dfstodfs0(dfsfile, dfs0file, func=np.nanmean): """ Function: take average (or other statistics) over dfs and output dfs0 Usage: dfstodfs0(dfsfile, dfs0file) dfstodfs0(dfsfile, dfs0file, func=np.nanmean) """ # Read dfs dfs_in = DfsFileFactory.DfsGenericOpen(dfsfile) fileInfo = dfs_in.FileInfo # Basic info from input file n_time_steps = fileInfo.TimeAxis.NumberOfTimeSteps if n_time_steps == 0: raise Warning( "Static dfs files (with no time steps) are not supported.") # Create an empty dfs1 file object factory = DfsFactory() builder = DfsBuilder.Create(fileInfo.FileTitle, fileInfo.ApplicationTitle, fileInfo.ApplicationVersion) # Set up the header builder.SetDataType(fileInfo.DataType) builder.SetGeographicalProjection(factory.CreateProjectionUndefined()) builder.SetTemporalAxis(fileInfo.TimeAxis) builder.DeleteValueByte = fileInfo.DeleteValueByte builder.DeleteValueDouble = fileInfo.DeleteValueDouble builder.DeleteValueFloat = fileInfo.DeleteValueFloat builder.DeleteValueInt = fileInfo.DeleteValueInt builder.DeleteValueUnsignedInt = fileInfo.DeleteValueUnsignedInt # dynamic items n_items = safe_length(dfs_in.ItemInfo) for item in range(n_items): ii = dfs_in.ItemInfo[item] itemj = builder.CreateDynamicItemBuilder() itemj.Set(ii.Name, ii.Quantity, DfsSimpleType.Float) itemj.SetValueType(DataValueType.Instantaneous) itemj.SetAxis(factory.CreateAxisEqD0()) # itemj.SetReferenceCoordinates(0, 0, 0) builder.AddDynamicItem(itemj.GetDynamicItemInfo()) try: builder.CreateFile(dfs0file) except IOError: print("cannot create dfs0 file: ", dfs0file) dfs_out = builder.GetFile() # read-write data deleteValue = fileInfo.DeleteValueFloat for it in range(n_time_steps): for item in range(n_items): itemdata = dfs_in.ReadItemTimeStep(item + 1, it) d = to_numpy(itemdata.Data) d[d == deleteValue] = np.nan d0 = func(d) d = np.zeros(1) d[0] = d0 d[np.isnan(d)] = deleteValue darray = to_dotnet_float_array(d) dfs_out.WriteItemTimeStepNext(itemdata.Time, darray) dfs_in.Close() dfs_out.Close()
def dfs2todfs1(dfs2file, dfs1file, axis=1, func=np.nanmean): """Aggregate file over an axis Parameters ---------- dfs2file : str input file dfs1file : str output file axis : int, optional spatial axis to aggregate over, 1=y, 2=x default 1 func : function, optional aggregation function, by default np.nanmean """ if axis not in [1, 2]: raise ValueError("Axis must be 1=y or 2=x") # Read dfs2 dfs_in = DfsFileFactory.DfsGenericOpen(dfs2file) fileInfo = dfs_in.FileInfo # Basic info from input file ax = dfs_in.ItemInfo[0].SpatialAxis n_time_steps = fileInfo.TimeAxis.NumberOfTimeSteps if n_time_steps == 0: raise Warning( "Static dfs2 files (with no time steps) are not supported.") # Create an empty dfs1 file object factory = DfsFactory() builder = Dfs1Builder.Create(fileInfo.FileTitle, fileInfo.ApplicationTitle, fileInfo.ApplicationVersion) # Set up the header builder.SetDataType(fileInfo.DataType) builder.SetGeographicalProjection(fileInfo.Projection) builder.SetTemporalAxis(fileInfo.TimeAxis) builder.DeleteValueByte = fileInfo.DeleteValueByte builder.DeleteValueDouble = fileInfo.DeleteValueDouble builder.DeleteValueFloat = fileInfo.DeleteValueFloat builder.DeleteValueInt = fileInfo.DeleteValueInt builder.DeleteValueUnsignedInt = fileInfo.DeleteValueUnsignedInt if axis == 1: builder.SetSpatialAxis( factory.CreateAxisEqD1(ax.AxisUnit, ax.XCount, ax.X0, ax.Dx)) else: builder.SetSpatialAxis( factory.CreateAxisEqD1(ax.AxisUnit, ax.YCount, ax.Y0, ax.Dy)) # assume no compression keys if fileInfo.IsFileCompressed: raise Warning("Compressed files not supported") # custom blocks # cb = fileInfo.CustomBlocks #[0] # for j in range(safe_length(cb)): # builder.AddCustomBlocks(cb[j]) # static items while True: static_item = dfs_in.ReadStaticItemNext() if static_item == None: break builder.AddStaticItem(static_item) # dynamic items n_items = safe_length(dfs_in.ItemInfo) for item in range(n_items): ii = dfs_in.ItemInfo[item] builder.AddDynamicItem(ii.Name, ii.Quantity, DfsSimpleType.Float, DataValueType.Instantaneous) try: builder.CreateFile(dfs1file) except IOError: print("cannot create dfs1 file: ", dfs1file) dfs_out = builder.GetFile() # read-write data deleteValue = fileInfo.DeleteValueFloat for it in range(n_time_steps): for item in range(n_items): itemdata = dfs_in.ReadItemTimeStep(item + 1, it) d = to_numpy(itemdata.Data) d[d == deleteValue] = np.nan d2 = d.reshape(ax.YCount, ax.XCount) d2 = np.flipud(d2) d1 = func(d2, axis=axis - 1) d1[np.isnan(d1)] = deleteValue darray = to_dotnet_float_array(d1) dfs_out.WriteItemTimeStepNext(itemdata.Time, darray) dfs_in.Close() dfs_out.Close()
def read(self, filename, item_numbers=None): """ Function: Read a dfs1 file usage: [data, time, name] = read(filename, item_numbers) item_numbers is a list of indices (base 0) to read from Returns 1) the data contained in a dfs1 file in a list of numpy matrices 2) time index 3) name of the items NOTE Returns data (nt, x) """ # NOTE. Item numbers are base 0 (everything else in the dfs is base 0) # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(filename) if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis xNum = axis.XCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if nt == 0: raise Warning( "Static dfs1 files (with no time steps) are not supported.") nt = 1 deleteValue = dfs.FileInfo.DeleteValueFloat n_items = len(item_numbers) data_list = [] for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(nt, xNum), dtype=float) data_list.append(data) t = [] startTime = dfs.FileInfo.TimeAxis.StartDateTime for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) d[d == deleteValue] = np.nan data_list[item][it, :] = d t.append( startTime.AddSeconds( itemdata.Time).ToString("yyyy-MM-dd HH:mm:ss")) time = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in t] names = [] for item in range(n_items): name = dfs.ItemInfo[item].Name names.append(name) dfs.Close() return Dataset(data_list, time, names)
def read(self, filename, item_numbers=None, item_names=None): """Read data from the dfs1 file Usage: read(filename, item_numbers=None, item_names=None) filename full path to the dfs1 file. item_numbers read only the item_numbers in the array specified (0 base) item_names read only the items in the array specified, (takes precedence over item_numbers) Return: Dataset(data, time, items) where data[nt,x] """ # NOTE. Item numbers are base 0 (everything else in the dfs is base 0) # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(filename) if item_names is not None: item_numbers = find_item(dfs, item_names) if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis xNum = axis.XCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if nt == 0: raise Warning("Static dfs1 files (with no time steps) are not supported.") nt = 1 deleteValue = dfs.FileInfo.DeleteValueFloat n_items = len(item_numbers) data_list = [] for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(nt, xNum), dtype=float) data_list.append(data) t_seconds = np.zeros(nt, dtype=float) for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) d[d == deleteValue] = np.nan data_list[item][it, :] = d t_seconds[it] = itemdata.Time start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] items = get_item_info(dfs, item_numbers) dfs.Close() return Dataset(data_list, time, items)
def concat(infilenames, outfilename): """Concatenates files along the time axis If files are overlapping, the last one will be used. Parameters ---------- infilenames: list of str filenames to concatenate outfilename: str filename Notes ------ The list of input files have to be sorted, i.e. in chronological order """ dfs_i_a = DfsFileFactory.DfsGenericOpen(infilenames[0]) dfs_o = _clone(infilenames[0], outfilename) n_items = safe_length(dfs_i_a.ItemInfo) for i, infilename in enumerate(infilenames): dfs_i = DfsFileFactory.DfsGenericOpen(infilename) n_time_steps = dfs_i.FileInfo.TimeAxis.NumberOfTimeSteps dt = dfs_i.FileInfo.TimeAxis.TimeStep start_time = from_dotnet_datetime(dfs_i.FileInfo.TimeAxis.StartDateTime) if i > 0 and start_time > current_time + timedelta(seconds=dt): dfs_o.Close() os.remove(outfilename) raise Exception("Gap in time axis detected - not supported") current_time = start_time if i < (len(infilenames) - 1): dfs_n = DfsFileFactory.DfsGenericOpen(infilenames[i + 1]) nf = dfs_n.FileInfo.TimeAxis.StartDateTime next_start_time = datetime( nf.Year, nf.Month, nf.Day, nf.Hour, nf.Minute, nf.Second ) for timestep in range(n_time_steps): current_time = start_time + timedelta(seconds=timestep * dt) if i < (len(infilenames) - 1): if current_time >= next_start_time: break for item in range(n_items): itemdata = dfs_i.ReadItemTimeStep(item + 1, timestep) d = to_numpy(itemdata.Data) darray = to_dotnet_float_array(d) dfs_o.WriteItemTimeStepNext(0, darray) dfs_o.Close()
def read(self, dfs3file, item_numbers=None, layers=None, coordinates=None): """ Function: Read data from a dfs3 file Usage: [data,time,name] = read( filename, item_numbers, layers=None, coordinates=None) item_numbers list of indices (base 0) to read from. If None then all the items. layers list of layer indices (base 0) to read coordinates list of list (x,y,layer) integers ( 0,0 at Bottom Left of Grid !! ) example coordinates = [[2,5,1], [11,41,2]] Returns 1) the data contained in a dfs3 file in a list of numpy matrices 2) time index 3) name of the items NOTE Returns data ( y, x, z, nt) 1) If coordinates is selected, then only return data at those coordinates 2) coordinates specified overules layers. 3) layer counts from the bottom """ # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(dfs3file) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis zNum = axis.ZCount yNum = axis.YCount xNum = axis.XCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps deleteValue = dfs.FileInfo.DeleteValueFloat if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) n_items = len(item_numbers) data_list = [] if coordinates is None: # if nt is 0, then the dfs is 'static' and must be handled differently if nt != 0: for item in range(n_items): if layers is None: # Initialize an empty data block data = np.ndarray(shape=(yNum, xNum, zNum, nt), dtype=float) # .fill(deleteValue) data_list.append(data) else: data = np.ndarray(shape=(yNum, xNum, len(layers), nt), dtype=float) # .fill(deleteValue) data_list.append(data) else: raise Warning( "Static dfs3 files (with no time steps) are not supported." ) quit() else: ncoordinates = len(coordinates) for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(ncoordinates, nt), dtype=float) data_list.append(data) t = [] startTime = dfs.FileInfo.TimeAxis.StartDateTime if coordinates is None: for it in range(nt): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) # DO a direct copy instead of eleement by elment d = d.reshape(zNum, yNum, xNum).swapaxes(0, 2).swapaxes(0, 1) d = np.flipud(d) d[d == deleteValue] = np.nan if layers is None: data_list[item][:, :, :, it] = d else: for l in range(len(layers)): data_list[item][:, :, l, it] = d[:, :, layers[l]] t.append( startTime.AddSeconds( itemdata.Time).ToString("yyyy-MM-dd HH:mm:ss")) else: indices = [ self.__calculate_index(xNum, yNum, zNum, x, y, z) for x, y, z in coordinates ] for it in range(nt): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) d = np.array([itemdata.Data[i] for i in indices]) d[d == deleteValue] = np.nan data_list[item][:, it] = d t.append( startTime.AddSeconds( itemdata.Time).ToString("yyyy-MM-dd HH:mm:ss")) # time = pd.DatetimeIndex(t) time = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in t] names = [] for item in range(n_items): name = dfs.ItemInfo[item_numbers[item]].Name names.append(name) dfs.Close() return Dataset(data_list, time, names)
def dfs2todfs1(dfs2file, dfs1file, ax=0, func=np.nanmean): """ Function: take average (or other statistics) over axis in dfs2 and output to dfs1 Usage: dfs2todfs1(dfs2file, dfs1file) dfs2todfs1(dfs2file, dfs1file, axis) dfs2todfs1(dfs2file, dfs1file, axis, func=np.nanmean) """ # Read dfs2 dfs_in = DfsFileFactory.DfsGenericOpen(dfs2file) fileInfo = dfs_in.FileInfo # Basic info from input file axis = dfs_in.ItemInfo[0].SpatialAxis n_time_steps = fileInfo.TimeAxis.NumberOfTimeSteps if n_time_steps == 0: raise Warning( "Static dfs2 files (with no time steps) are not supported.") # Create an empty dfs1 file object factory = DfsFactory() builder = Dfs1Builder.Create(fileInfo.FileTitle, fileInfo.ApplicationTitle, fileInfo.ApplicationVersion) # Set up the header builder.SetDataType(fileInfo.DataType) builder.SetGeographicalProjection(fileInfo.Projection) builder.SetTemporalAxis(fileInfo.TimeAxis) builder.DeleteValueByte = fileInfo.DeleteValueByte builder.DeleteValueDouble = fileInfo.DeleteValueDouble builder.DeleteValueFloat = fileInfo.DeleteValueFloat builder.DeleteValueInt = fileInfo.DeleteValueInt builder.DeleteValueUnsignedInt = fileInfo.DeleteValueUnsignedInt # use x-axis (default) else y-axis if ax == 0: builder.SetSpatialAxis( factory.CreateAxisEqD1(axis.AxisUnit, axis.XCount, axis.X0, axis.Dx)) else: builder.SetSpatialAxis( factory.CreateAxisEqD1(axis.AxisUnit, axis.YCount, axis.Y0, axis.Dy)) # assume no compression keys if fileInfo.IsFileCompressed: raise Warning("Compressed files not supported") # custom blocks # cb = fileInfo.CustomBlocks #[0] # for j in range(safe_length(cb)): # builder.AddCustomBlocks(cb[j]) # static items while True: static_item = dfs_in.ReadStaticItemNext() if static_item == None: break builder.AddStaticItem(static_item) # dynamic items n_items = safe_length(dfs_in.ItemInfo) for item in range(n_items): ii = dfs_in.ItemInfo[item] builder.AddDynamicItem(ii.Name, ii.Quantity, DfsSimpleType.Float, DataValueType.Instantaneous) try: builder.CreateFile(dfs1file) except IOError: print("cannot create dfs1 file: ", dfs1file) dfs_out = builder.GetFile() # read-write data deleteValue = fileInfo.DeleteValueFloat for it in range(n_time_steps): for item in range(n_items): itemdata = dfs_in.ReadItemTimeStep(item + 1, it) d = to_numpy(itemdata.Data) d[d == deleteValue] = np.nan d2 = d.reshape(axis.YCount, axis.XCount) d2 = np.flipud(d2) d1 = func(d2, axis=ax) d1[np.isnan(d1)] = deleteValue darray = Array[System.Single](d1) dfs_out.WriteItemTimeStepNext(itemdata.Time, darray) dfs_in.Close() dfs_out.Close()
def read(self, filename, item_numbers=None, item_names=None): """Read data from the dfs1 file Usage: read(filename, item_numbers=None, item_names=None) filename full path to the dfs1 file. item_numbers read only the item_numbers in the array specified (0 base) item_names read only the items in the array specified, (takes precedence over item_numbers) Return: Dataset(data, time, items) where data[nt,y,x] """ # NOTE. Item numbers are base 0 (everything else in the dfs is base 0) # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(filename) if item_names is not None: item_numbers = find_item(dfs, item_names) if item_numbers is None: n_items = safe_length(dfs.ItemInfo) item_numbers = list(range(n_items)) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis yNum = axis.YCount xNum = axis.XCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if nt == 0: raise Warning( "Static files (with no dynamic items) are not supported.") nt = 1 deleteValue = dfs.FileInfo.DeleteValueFloat n_items = len(item_numbers) data_list = [] for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(nt, yNum, xNum), dtype=float) data_list.append(data) t = [] startTime = dfs.FileInfo.TimeAxis.StartDateTime for it in range(dfs.FileInfo.TimeAxis.NumberOfTimeSteps): for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) d = d.reshape(yNum, xNum) d = np.flipud(d) d[d == deleteValue] = np.nan data_list[item][it, :, :] = d t.append( startTime.AddSeconds( itemdata.Time).ToString("yyyy-MM-dd HH:mm:ss")) time = [datetime.strptime(x, "%Y-%m-%d %H:%M:%S") for x in t] items = get_item_info(dfs, item_numbers) dfs.Close() return Dataset(data_list, time, items)
def read(self, items=None, time_steps=None): """ Read data from a dfs1 file Parameters --------- items: list[int] or list[str], optional Read only selected items, by number (0-based), or by name time_steps: int or list[int], optional Read only selected time_steps Returns ------- Dataset A dataset with data dimensions [t,x] """ # NOTE. Item numbers are base 0 (everything else in the dfs is base 0) # Open the dfs file for reading dfs = DfsFileFactory.DfsGenericOpen(self._filename) self._dfs = dfs self._source = dfs nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps items, item_numbers, time_steps = get_valid_items_and_timesteps( self, items, time_steps) # Determine the size of the grid axis = dfs.ItemInfo[0].SpatialAxis xNum = axis.XCount nt = dfs.FileInfo.TimeAxis.NumberOfTimeSteps if nt == 0: raise ValueError( "Static dfs1 files (with no time steps) are not supported.") deleteValue = dfs.FileInfo.DeleteValueFloat n_items = len(item_numbers) data_list = [] for item in range(n_items): # Initialize an empty data block data = np.ndarray(shape=(len(time_steps), xNum), dtype=float) data_list.append(data) t_seconds = np.zeros(len(time_steps), dtype=float) for i in range(len(time_steps)): it = time_steps[i] for item in range(n_items): itemdata = dfs.ReadItemTimeStep(item_numbers[item] + 1, it) src = itemdata.Data d = to_numpy(src) d[d == deleteValue] = np.nan data_list[item][it, :] = d t_seconds[it] = itemdata.Time start_time = from_dotnet_datetime(dfs.FileInfo.TimeAxis.StartDateTime) time = [start_time + timedelta(seconds=tsec) for tsec in t_seconds] items = get_item_info(dfs, item_numbers) dfs.Close() return Dataset(data_list, time, items)