def dist(self, pfb_file, **kwargs): """Distribute a PFB file using the P/Q/R settings from the run or override them with the provided arguments. We can also use the kwargs to set other properties such as: - NX, NY, NZ... """ # Any provided args should override the scripts ones update_run_from_args(self, self._process_args_) pfb_file_full_path = get_absolute_path(pfb_file) p = self.Process.Topology.P q = self.Process.Topology.Q r = self.Process.Topology.R if 'P' in kwargs: p = kwargs['P'] if 'Q' in kwargs: q = kwargs['Q'] if 'R' in kwargs: r = kwargs['R'] from parflowio.pyParflowio import PFData pfb_data = PFData(pfb_file_full_path) pfb_data.distFile(p, q, r, pfb_file_full_path)
def test_copy(self): data = np.random.random_sample((50, 49, 31)) test = PFData(data) copy = test.copyDataArray() self.assertTrue( np.array_equal(data, copy), 'Data obtained from PFData::copyDataArray must match given data')
def write_pfb_output(forcings_data, num_days, out_dir, dx, start_day_num=0): varnames = ['APCP', 'DLWR', 'DSWR', 'Press', 'SPFH', 'Temp', 'UGRD', 'VGRD'] hours_in_file = 24 for var in tqdm(varnames): file_time_start = start_day_num * hours_in_file file_time_stop = file_time_start + hours_in_file start = 0 # start hour (inclusive) stop = hours_in_file # end hour (exclusive) # start hour and stop hour for a day # range is number of days contained in forcings file for bulk_collect_times in range(0, num_days): data_obj = PFData(np.nan_to_num(forcings_data[var].values[start:stop, :, :], nan=-9999.0)) data_obj.setDX(dx) data_obj.setDY(dx) data_obj.setDZ(dx) data_obj.writeFile(os.path.join(out_dir, f'WRF.{var}.{file_time_start+1:06d}_to_{file_time_stop:06d}.pfb')) del data_obj start = stop stop = stop + hours_in_file # size of day in hours file_time_start = file_time_stop file_time_stop = file_time_stop + hours_in_file
def test_view(self): data = np.random.random_sample((50, 49, 31)) test = PFData(data) view = test.viewDataArray() self.assertTrue( np.array_equal(data, view), 'Data obtained from PFData::viewDataArray must match given data')
def test_move(self): data = np.random.random_sample((50, 49, 31)) test = PFData(data) move = test.moveDataArray() self.assertTrue( np.array_equal(data, move), 'Data obtained from PFData::moveDataArray must match given data') self.assertIsNone( test.viewDataArray(), 'Calling PFData::moveDataArray must invalidate the internal data pointer' )
def write_array_pfb(file_name, array): # Ensure this is 3 dimensions, since parflowio requires 3 dimensions. while array.ndim < 3: array = array[np.newaxis, :] if array.ndim > 3: raise Exception(f'Too many dimensions: {array.ndim}') from parflowio.pyParflowio import PFData data = PFData() data.setDataArray(array) return data.writeFile(file_name)
def pfread(pfbfile): """ Read a pfb file and return data as an ndarray :param pfbfile: path to pfb file :return: An ndarray of ndim=3, with shape (nz, ny, nx) """ if not os.path.exists(pfbfile): raise RuntimeError(f'{pfbfile} not found') pfb_data = PFData(pfbfile) pfb_data.loadHeader() pfb_data.loadData() arr = pfb_data.moveDataArray() pfb_data.close() assert arr.ndim == 3, 'Only 3D arrays are supported' return arr
def _pfb_to_array(self, file_path): from parflowio.pyParflowio import PFData array = None if file_path: full_path = get_absolute_path(file_path) # FIXME do something with selector inside parflow-io pfb_data = PFData(full_path) pfb_data.loadHeader() pfb_data.loadData() array = pfb_data.moveDataArray() return array
def read_file(infile): """read an input file and return a 3d numpy array Parameters ---------- infile : str file to open (.pfb, .sa, .tif, .tiff) Returns ------- res_arr : ndarray a 3d numpy array with data from file in (z,y,x) format with y axis 0 at bottom """ infile_path = Path(infile) # get extension ext = infile_path.suffix file_string_path = os.fspath(infile_path) if ext in ['.tif', '.tiff']: res_arr = gdal.Open(file_string_path).ReadAsArray() if len(res_arr.shape) == 2: res_arr = res_arr[np.newaxis, ...] # flip y axis so tiff aligns with PFB native alignment res_arr = np.flip(res_arr, axis=1) elif ext == '.sa': # parflow ascii file with open(file_string_path, 'r') as fi: header = fi.readline() nx, ny, nz = [int(x) for x in header.strip().split(' ')] arr = pd.read_csv(file_string_path, skiprows=1, header=None).values res_arr = np.reshape(arr, (nz, ny, nx))[:, :, :] elif ext == '.pfb': # parflow binary file pfdata = PFData(file_string_path) pfdata.loadHeader() pfdata.loadData() res_arr = pfdata.moveDataArray() pfdata.close() del pfdata else: raise ValueError('can not read file type ' + ext) return res_arr
def load_patch_matrix_from_pfb_file(file_name, layer=None): from parflowio.pyParflowio import PFData data = PFData(file_name) data.loadHeader() data.loadData() data_array = data.viewDataArray() if data_array.ndim == 3: nlayer, nrows, ncols = data_array.shape if layer: nlayer = layer return data_array[nlayer - 1, :, :] elif data_array.ndim == 2: return data_array else: raise Exception(f'invalid PFB file: {file_name}')
def test_validate_cell_values(self): test = PFData(('press.init.pfb')) retval = test.loadHeader() self.assertEqual(0, retval, 'should load header of file that exists') retval = test.loadData() self.assertEqual(0, retval, 'should load data from valid file') data = test.getDataAsArray() self.assertIsNotNone( data, 'data from object should be available as python object') self.assertSequenceEqual((50, 41, 41), data.shape) self.assertAlmostEqual(98.003604098773, test(0, 0, 0), 12, 'valid data in cell (0,0,0)') self.assertAlmostEqual(97.36460429313328, test(40, 0, 0), 12, 'data in cell (40,0,0)') self.assertAlmostEqual(98.0043134691891, test(0, 1, 0), 12, 'data in cell (0, 1, 0)') self.assertAlmostEqual(98.00901307022781, test(1, 0, 0), 12, 'data in cell (1, 0, 0)') self.assertAlmostEqual(92.61370155558751, test(21, 1, 2), 12, 'data in cell (21, 1, 2)') self.assertAlmostEqual(7.98008728357588, test(0, 1, 45), 12, 'data in cell (0, 1, 45)') self.assertAlmostEqual(97.30205516102234, test(22, 1, 0), 12, 'valid data in cell (22,1,0)') self.assertEqual(test(0, 0, 0), data[0, 0, 0], 'data array and c array match values at (0,0,0)') self.assertEqual(test(40, 0, 0), data[0, 0, 40], 'data array and c array match values at (40,0,0)') self.assertEqual(test(0, 1, 0), data[0, 1, 0], 'data array and c array match values at (0,1,0)') self.assertEqual(test(1, 0, 0), data[0, 0, 1], 'data array and c array match values at (1,0,0)') self.assertEqual(test(21, 1, 2), data[2, 1, 21], 'data array and c array match values at (21,1,2)') self.assertEqual(test(0, 1, 45), data[45, 1, 0], 'data array and c array match values at (0,1,45)') self.assertEqual(test(22, 1, 0), data[0, 1, 22], 'data array and c array match values at (22,1,0)') test.close()
def test_bad_filename(self): test = PFData('bad_filename_not_exists.pfb') retval = test.loadHeader() self.assertNotEqual(0, retval, 'Bad filename should error loading header')
def test_empty_init(self): test = PFData() retval = test.loadHeader() self.assertNotEqual(0, retval, 'Empty class should error loading header')
def test_create_from_data(self): data = np.random.random_sample((50, 49, 31)) test = PFData(data) self.assertEqual(31, test.getNX()) self.assertEqual(49, test.getNY()) self.assertEqual(50, test.getNZ()) self.assertEqual(1, test.getP()) self.assertEqual(1, test.getQ()) self.assertEqual(1, test.getR()) self.assertEqual(0, test.getX()) self.assertEqual(0, test.getY()) self.assertEqual(0, test.getZ()) test.writeFile(('test_write_raw.pfb')) test_read = PFData(('test_write_raw.pfb')) test_read.loadHeader() test_read.loadData() self.assertEqual(0, test_read.getX()) self.assertEqual(0, test_read.getY()) self.assertEqual(0, test_read.getZ()) self.assertEqual(31, test_read.getNX()) self.assertEqual(49, test_read.getNY()) self.assertEqual(50, test_read.getNZ()) self.assertEqual(1, test_read.getP()) self.assertEqual(1, test_read.getQ()) self.assertEqual(1, test_read.getR()) test_data = test_read.getDataAsArray() self.assertIsNone( np.testing.assert_array_equal(data, test_data), 'Data written to array should exist in ' 'written PFB file.') del data test.close() test_read.close() os.remove(('test_write_raw.pfb'))
def test_dist_nldas_file(self): test = PFData(('NLDAS.APCP.000001_to_000024.pfb')) test.distFile(P=2, Q=2, R=1, outFile=('NLDAS.APCP.000001_to_000024.pfb.tmp')) out_file = PFData(('NLDAS.APCP.000001_to_000024.pfb.tmp')) dist_file = open(('NLDAS.APCP.000001_to_000024.pfb.tmp.dist'), 'r') dist_lines = dist_file.readlines() [ self.assertEqual(int(line.rstrip('\n')), val) for line, val in zip( dist_lines, [0, 84772, 165448, 246124, 322960]) ] self.assertEqual(0, out_file.loadHeader(), 'should load distributed file header') self.assertEqual(0, out_file.loadData(), 'should load distributed data') self.assertIsNone( np.testing.assert_array_equal(test.getDataAsArray(), out_file.getDataAsArray()), 'should find matching data values in original and distributed files' ) test.close() out_file.close() dist_file.close() os.remove(('NLDAS.APCP.000001_to_000024.pfb.tmp')) os.remove(('NLDAS.APCP.000001_to_000024.pfb.tmp.dist'))
def test_dist_file(self): test = PFData(('press.init.pfb')) test.distFile(P=2, Q=2, R=1, outFile=('press.init.pfb.tmp')) out_file = PFData(('press.init.pfb.tmp')) dist_file = open(('press.init.pfb.tmp.dist'), 'r') dist_lines = dist_file.readlines() [ self.assertEqual(int(line.rstrip('\n')), val) for line, val in zip( dist_lines, [0, 176500, 344536, 512572, 672608]) ] self.assertEqual(0, out_file.loadHeader(), 'should load distributed file header') self.assertEqual(0, out_file.loadData(), 'should load distributed data') self.assertIsNone( np.testing.assert_array_equal(test.getDataAsArray(), out_file.getDataAsArray()), 'should find matching data values in original and distributed files' ) test.close() out_file.close() dist_file.close() os.remove(('press.init.pfb.tmp')) os.remove(('press.init.pfb.tmp.dist'))
def test_manipulate_data(self): test = PFData(('press.init.pfb')) retval = test.loadHeader() self.assertEqual(0, retval, 'should load header of file that exists') retval = test.loadData() self.assertEqual(0, retval, 'should load data from valid file') test_data = test.getDataAsArray() self.assertSequenceEqual( (50, 41, 41), test_data.shape, 'test file array should have shape (50,41,41)') self.assertAlmostEqual(98.003604098773, test(0, 0, 0), 12, 'valid data in cell (0,0,0)') test_data[0, 0, 0] = 1 test_data[0, 0, 40] = 1 test_data[2, 1, 21] = 1 self.assertEqual(1, test(0, 0, 0), 'data update affects underlying array') self.assertEqual(1, test(40, 0, 0), 'data update affects underlying array') self.assertEqual(1, test(21, 1, 2), 'data update affects underlying array') retval = test.writeFile(('press.init.pfb.tmp')) self.assertEqual(0, retval, 'able to write updated data to output file') test_read = PFData(('press.init.pfb.tmp')) test_read.loadHeader() test_read.loadData() self.assertEqual(1, test_read(0, 0, 0), 'updates to data written to file can be read back') self.assertEqual(1, test_read(40, 0, 0), 'updates to data written to file can be read back') self.assertEqual(1, test_read(21, 1, 2), 'updates to data written to file can be read back') test.close() test_read.close() os.remove(('press.init.pfb.tmp'))
def test_load_data_threaded_perf(self): # loadData - Not using threads non_threaded_time = 0 for i in range(10000): base = PFData( ('LW.out.press.00000.pfb' )) # Little Washita pressure PFB with 2x2x1 grid size base.loadHeader() start = time.time_ns() base.loadData() non_threaded_time += time.time_ns() - start # loadDataThreaded - Using 4 threads num_threads = 4 threaded_time = 0 for i in range(10000): test = PFData( ('LW.out.press.00000.pfb' )) # Little Washita pressure PFB with 2x2x1 grid size test.loadHeader() start = time.time_ns() test.loadPQR( ) # loadPQR() must be called before loadDataThreaded() test.loadDataThreaded(num_threads) threaded_time += time.time_ns() - start base.close() test.close() # loadDataThreaded() should have less total time spent than loadData() self.assertTrue( threaded_time < non_threaded_time, f'Using {num_threads} threads has degraded the performance of loadDataThreaded()' ) # Display performance increase in percent change pct_change = 100 * abs(threaded_time - non_threaded_time) / non_threaded_time print( f'{pct_change:.2f}% performance increase when using LoadDataThreaded() with {num_threads} threads' )
def test_compare(self): test1 = PFData(('press.init.pfb')) test1.loadHeader() test1.loadData() test2 = PFData(('press.init.pfb')) test2.loadHeader() test2.loadData() self.assertEqual(PFData.differenceType_none, test1.compare(test2)[0], "test1 and test2 are the same") test1.setX(test1.getX() + 1.0) self.assertEqual(PFData.differenceType_x, test1.compare(test2)[0], "The x values differ") test1.setX(test1.getX() - 1.0) arr = test1.getDataAsArray() arr[1][2][3] += 1.0 ret, zyx = test1.compare(test2) self.assertEqual(PFData.differenceType_data, ret, "The data values differ") self.assertEqual((1, 2, 3), zyx, "The differing data's coordinates are correct") arr[1][2][3] -= 1.0 test1.close() test2.close()
ny = sandtank.ComputationalGrid.NY nz = sandtank.ComputationalGrid.NZ ## write flow boundary file flux_file_names = ['dry', 'rainflux_all', 'rainflux_left', 'rainflux_right'] for name in flux_file_names: array = np.full((nz, ny, nx), 0.0) if name == 'rainflux_all': array[(nz - 1), :, :] = rain_flux * 2 if name == 'rainflux_left': array[(nz - 1), :, 0:49] = rain_flux if name == 'rainflux_right': array[(nz - 1), :, 50:nx] = rain_flux pfb = PFData(array) pfb.writeFile(get_absolute_path(f'{name}.pfb')) pfb.close() sandtank.dist(f'{name}.pfb') # ----------------------------------------------------------------------------- # Boundary Condition definitions # ----------------------------------------------------------------------------- sandtank.BCPressure.PatchNames = sandtank.Geom.domain.Patches sandtank.Patch.x_lower.BCPressure.Type = 'DirEquilRefPatch' sandtank.Patch.x_lower.BCPressure.Cycle = 'constant' sandtank.Patch.x_lower.BCPressure.RefGeom = 'domain' sandtank.Patch.x_lower.BCPressure.RefPatch = 'z_lower'
def read_array_pfb(file_name): from parflowio.pyParflowio import PFData data = PFData(file_name) data.loadHeader() data.loadData() return data.moveDataArray()
def test_load_data_threaded(self): base = PFData(('press.init.pfb')) base.loadHeader() base.loadData() # 1 thread test1 = PFData(('press.init.pfb')) test1.loadHeader() test1.loadPQR() test1.loadDataThreaded(1) self.assertEqual(PFData.differenceType_none, base.compare(test1)[0], "base and test1 are the same") # 8 threads test8 = PFData(('press.init.pfb')) test8.loadHeader() test8.loadPQR() test8.loadDataThreaded(8) self.assertEqual(PFData.differenceType_none, base.compare(test8)[0], "base and test8 are the same") # 40 threads (more than the number of subgrids) test40 = PFData(('press.init.pfb')) test40.loadHeader() test40.loadPQR() test40.loadDataThreaded(40) self.assertEqual(PFData.differenceType_none, base.compare(test40)[0], "base and test40 are the same") base.close() test1.close() test8.close() test40.close()
def test_open_close(self): test = PFData() self.assertIsNone(test.close(), 'should be able to open and close an empty object')
def test_read_write_data(self): test = PFData(('press.init.pfb')) retval = test.loadHeader() self.assertEqual(0, retval, 'should load header of file that exists') retval = test.loadData() self.assertEqual(0, retval, 'should load data from valid file') retval = test.writeFile(('press.init.pfb.tmp')) self.assertEqual(0, retval, 'should write data from previously loaded file') data2 = PFData(('press.init.pfb.tmp')) data2.loadHeader() data2.loadData() self.assertIsNone( np.testing.assert_array_equal( test.getDataAsArray(), data2.getDataAsArray(), 'should read back same values we wrote')) in_file_hash = calculate_sha1_hash(('press.init.pfb')) out_file_hash = calculate_sha1_hash(('press.init.pfb.tmp')) # This assertion (that the files are identical) is failing in Python and in C++ # because the original test input file was written by a tool that incorrectly set the value self.assertNotEqual( in_file_hash, out_file_hash, 'sha1 hash of input and output files should not match') same, byte_diff = byte_compare_files(('press.init.pfb'), ('press.init.pfb.tmp')) self.assertFalse( same, 'press.init.pfb should differ from version just written') self.assertEqual(92, byte_diff, 'first byte difference at byte 92') test.close() data2.close() os.remove(('press.init.pfb.tmp'))
def test_good_filename(self): test = PFData(('press.init.pfb')) retval = test.loadHeader() self.assertEqual(0, retval, 'should load header of file that exists') self.assertEqual(41, test.getNX(), 'sample file should have 41 columns') self.assertEqual(41, test.getNY(), 'sample file should have 41 rows') self.assertEqual(50, test.getNZ(), 'sample file should have 50 z-layers') self.assertEqual(0, test.getX(), 'sample file starts at X=0') self.assertEqual(0, test.getY(), 'sample file starts at Y=0') self.assertEqual(0, test.getZ(), 'sample file starts at Z=0') self.assertEqual(16, test.getNumSubgrids(), 'sample file should have 16 subgrids') test.close()
def test_set_index_order(self): test = PFData(('press.init.pfb')) self.assertEqual(test.getIndexOrder(), 'zyx', 'indexOrder should equal \'zyx\'') test.setIndexOrder('xyz') self.assertEqual(test.getIndexOrder(), 'xyz', 'indexOrder should equal \'xyz\'') test.setIndexOrder('xYz') self.assertEqual(test.getIndexOrder(), 'xyz', 'indexOrder should equal \'xyz\'') test.setIndexOrder('xYZ') self.assertEqual(test.getIndexOrder(), 'xyz', 'indexOrder should equal \'xyz\'') test.setIndexOrder('XYZ') self.assertEqual(test.getIndexOrder(), 'xyz', 'indexOrder should equal \'xyz\'') test.setIndexOrder('XYZZZZ') self.assertEqual(test.getIndexOrder(), 'xyz', 'indexOrder should equal \'xyz\'') # Should not work, should still equal 'xyz' test.setIndexOrder('abc') self.assertEqual(test.getIndexOrder(), 'xyz', 'indexOrder should equal \'xyz\'') # Should not be able to write to file when indexOrder == 'xyz' self.assertEqual( test.writeFile(('test_write_index_order.pfb')), 1, 'Should not be able to write to file when indexOrder == \'xyz\'') # Should equal 'zyx' test.setIndexOrder('ZYX') self.assertEqual(test.getIndexOrder(), 'zyx', 'indexOrder should equal \'zyx\'') # Should equal 'zyx' test.setIndexOrder('zYx') self.assertEqual(test.getIndexOrder(), 'zyx', 'indexOrder should equal \'zyx\'') # Should be able to write to file self.assertEqual( test.writeFile(('test_write_index_order.pfb')), 0, 'Should be able to write to file when indexOrder == \'zyx\'') # Read file, indexOrder should equal 'zyx' test_read = PFData(('test_write_index_order.pfb')) test_read.loadHeader() test_read.loadData() self.assertEqual(test.getIndexOrder(), 'zyx', 'indexOrder should equal \'zyx\'') test.close() test_read.close() os.remove(('test_write_index_order.pfb'))
def pfwrite(arr, pfbfile, dx=1, dy=1, dz=1, overwrite=False): """ Save an ndarray to a pfb file :param arr: ndarray to save (must be 3-dimensional) :param pfbfile: path to pfb file :param overwrite: whether to overwrite the file if it exists :return: None on success. Raises Exception on failure. """ if os.path.exists(pfbfile) and not overwrite: raise RuntimeError(f'{pfbfile} already exists') assert arr.ndim == 3, 'Only 3D arrays are supported' pfb_data = PFData() pfb_data.setDataArray(arr) pfb_data.setDX(dx) pfb_data.setDY(dy) pfb_data.setDZ(dz) pfb_data.writeFile(pfbfile) pfb_data.close()
# ax.set_title("frame {}".format(i)) # image.pyplot(plt) # #plt.pause(0.01) control2 = st.button(" Animate Results ") if control2 == True: N = 100 time = np.zeros( [N + 1]) # time array, we will probably want to swap with a date outflow = np.zeros([N + 1]) # array to load in the meterological forcing sat = np.zeros([N + 1, 300, 20]) for icount in range(0, N): base = (base_dir + "/dunne_over/Dunne.out.satur.{:05d}.pfb") filename = base.format(icount) data_obj = PFData(filename) data_obj.loadHeader() data_obj.loadData() data_arr = data_obj.getDataAsArray() data_obj.close() sat[icount, :, :] = np.where(data_arr[:, 0, :] <= 0.0, 0.0, data_arr[:, 0, :]) fig, ax = plt.subplots() image = st.pyplot(plt) for i in range(N): ax.cla() ax.imshow(sat[i, :, :], vmin=0.1, vmax=1.0, origin='lower',
# Flow Barrier in X between cells 10 and 11 in all Z #--------------------------------------------------------- rich_fbx.Solver.Nonlinear.FlowBarrierX = True rich_fbx.FBx.Type = 'PFBFile' rich_fbx.Geom.domain.FBx.FileName = 'Flow_Barrier_X.pfb' ## write flow boundary file FBx_data = np.full((20, 20, 20), 1.0) for i in range(20): for j in range(20): # from cell 10 (index 9) to cell 11 # reduction of 1E-3 FBx_data[i, j, 9] = 0.001 FBx_data_pfb = PFData(FBx_data) FBx_data_pfb.writeFile(get_absolute_path('Flow_Barrier_X.pfb')) FBx_data_pfb.close() rich_fbx.dist('Flow_Barrier_X.pfb') #----------------------------------------------------------------------------- # Wells #----------------------------------------------------------------------------- rich_fbx.Wells.Names = '' #----------------------------------------------------------------------------- # Time Cycles #-----------------------------------------------------------------------------
def write_pfb(data, outfile, x0=0, y0=0, z0=0, dx=1000, dz=1000): """Write a 3d numpy array to a PFB output file Parameters ---------- data : ndarray 3d numpy data array to write to pfb !(x,y,z)! outfile : str filename and path to write output x0 : int, optional initial x location (Default value = 0) y0 : int, optional initial y location (Default value = 0) z0 : int, optional initial z location (Default value = 0) dx : int, optional horizontal resolution (Default value = 1000) dz : int, optional vertical resolution (Default value = 1000) Returns ------- None """ logging.info(f'wrote pfb file {outfile}, (z,y,x)={data.shape}') pf_data = PFData() pf_data.setDataArray(data) pf_data.setDX(dx) pf_data.setDY(dx) pf_data.setDZ(dz) pf_data.setX(x0) pf_data.setY(y0) pf_data.setZ(z0) pf_data.writeFile(outfile)