def calc_slope_data(gauges_in_mask, slope_file_x, slope_file_y):
# calculate the slope for all the inbounds gauges as slope=sqrt(x.slope^2+y.slope^2)
slope_data_x = np.flip(pfio.pfread(slope_file_x), axis=1)
slope_data_y = np.flip(pfio.pfread(slope_file_y), axis=1)
return gauges_in_mask.assign(
slope_x=lambda row: slope_data_x[0, row.mapped_j, row.mapped_i],
slope_y=lambda row: slope_data_y[0, row.mapped_j, row.mapped_i],
slope=lambda row: (row.slope_x**2 + row.slope_y**2)**0.5)
def calculate_flow_data(gauges_in_mask, pressure_files):
# create a new empty dataframe to hold the flow data
flow_data = pd.DataFrame()
# iterate over all pressure files found and collect pressure information
for p_file in pressure_files:
top_layer = -1
ts = int(p_file.split('.')[-2])
pressure_data = np.flip(pfio.pfread(p_file), axis=1)
# print(pressure_data[top_layer, 502, 249])
# print(pressure_data.shape)
p_data = gauges_in_mask.assign(pressure=lambda row: pressure_data[
top_layer, row.mapped_j, row.mapped_i],
timestep=ts)
p_data = p_data.filter([
'STAID', 'STANAME', 'mapped_i', 'mapped_j', 'slope', 'timestep',
'pressure'
],
axis=1)
flow_data = flow_data.append(p_data, sort=False, ignore_index=True)
# do flow calculation
flow_data = flow_data.assign(
flow_cms=lambda row: RM * (row.slope.pow(.5)) *
(row.pressure.pow(5 / 3) / 3600),
flow_cfs=lambda row: row.flow_cms * CMS_TO_CFS)
# handle failure of flow calculation when press is negative
return flow_data.fillna(0)
def generate_flow_at_gauges(pf_outputs,
out_dir,
start_date=None,
print_png=False):
"""
calculate flow and save to csv from pressure file outputs of a ParFlow simulation.
expects mask, slope, pftcl, and subset extents files to be in pf_outputs along with pressure files
:param pf_outputs: directory path to the ParFlow simulation outputs
:param out_dir: directory for this code to write outputs to
:param start_date: optional starting date if outputs are daily
:param print_png: option to save PNG outputs along with csv outputs
:return: a pandas dataframe with flow for USGS gauge site inside the mask
"""
# make sure we have a pftcl file in the outputs directory
pftcl_file = find_pftcl_file(pf_outputs)
# parse the pftcl file's path and name to identify the ParFlow runname
runname = get_runname_from_pftcl(pftcl_file)
# get domain extents
domain_extents = find_subset_extents_file(pf_outputs, runname)
domain_extents = convert_y_extents(domain_extents)
subset_lower_left = (int(domain_extents[2]), int(domain_extents[0]))
subset_upper_right = (int(domain_extents[3]), int(domain_extents[1]))
# find the mask file and read it
check_mask_file_found(os.path.join(pf_outputs, f'{runname}.out.mask.pfb'))
mask_data = np.flip(pfio.pfread(
os.path.join(pf_outputs, f'{runname}.out.mask.pfb')),
axis=1)
# find the slope file names in the pftcl file
slope_file_x = parse_pftcl(pftcl_file, 'TopoSlopesX.FileName')
slope_file_y = parse_pftcl(pftcl_file, 'TopoSlopesY.FileName')
# read the list of gauges from the csv file
usgs_gauges = pd.read_csv(GAUGES_FILE)
# determine which gauges are inside the bounds of our domain
# filter the gauges by those that are inside our masked domain
gauges_in_extents = get_gauges_in_extents(subset_lower_left,
subset_upper_right, usgs_gauges)
gauges_in_mask = get_gauges_in_mask(subset_lower_left, mask_data,
gauges_in_extents)
pressure_files = get_pressure_files(pf_outputs, runname)
flow_data = get_flow_at_gauges(gauges_in_mask,
os.path.join(pf_outputs, slope_file_x),
os.path.join(pf_outputs, slope_file_y),
pressure_files)
if flow_data is not None:
csv_path = make_output_subdir(out_dir, 'csv')
write_flows_to_csv(flow_data, csv_path)
if print_png:
png_path = make_output_subdir(out_dir, 'png')
write_hydrographs_to_png(flow_data, png_path)
return flow_data
def preview(f):
arr = pfio.pfread(f)
# import pdb; pdb.set_trace()
# set 0 to null
arr[arr == 0.0] = numpy.nan
# set non-zero to 1
arr[~numpy.isnan(arr)] = 1
imgplot = plt.imshow(arr[0, :, :])
plt.show()
def test_something(self):
numpy_data = np.load('lw.press.init.npy')
input_pfb = 'press.init.pfb'
fort_pfb_data = np.zeros((41, 41, 50), order='F')
fort_io.pfb_read(fort_pfb_data, input_pfb)
fort_pfb_data = np.transpose(fort_pfb_data, (2, 1, 0))
c_pfb_data = pfio.pfread(input_pfb)
c_pfb_data = np.flip(c_pfb_data, axis=1)
self.assertIsNone(
np.testing.assert_array_equal(fort_pfb_data, c_pfb_data))
self.assertIsNone(
np.testing.assert_array_equal(fort_pfb_data, numpy_data))
self.assertIsNone(np.testing.assert_array_equal(
c_pfb_data, numpy_data))
def read_from_file(infile):
#get extension
ext = os.path.splitext(os.path.basename(infile))[1]
if ext in ['.tif', '.tiff']:
res_arr = gdal.Open(infile).ReadAsArray()
elif ext == '.sa': #parflow ascii file
with open(infile, 'r') as fi:
header = fi.readline()
nx, ny, nz = [int(x) for x in header.strip().split(' ')]
arr = pd.read_csv(infile, skiprows=1, header=None).values
res_arr = np.reshape(arr, (nz, ny, nx))[:, ::-1, :]
elif ext == '.pfb': #parflow binary file
res_arr = pfio.pfread(infile)
else:
print('can not read file type ' + ext)
sys.exit()
return res_arr
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jun 17 12:06:26 2020
@author: abramfarley
"""
import pfio
import numpy as np
from numpy import genfromtxt
import math
import os
import sys
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.lines as mlines
import matplotlib.cm as cm
import matplotlib as mpl
from scipy import pi
from scipy.fftpack import fft
from scipy import fftpack
from scipy import signal
from scipy import integrate
import statistics
from statsmodels.graphics.tsaplots import plot_acf
np.set_printoptions(threshold=sys.maxsize)
#%%
#csv of the input signal (white noise)
Signal = genfromtxt(
'/Users/abramfarley/ParF/uahpc/matching_inputs/over0.4freq_hn/harmonic_input_july.csv',
delimiter=',')
# finally change domain info
results['GeomInput.domaininput.FileName']['vals'][0][-1] = os.path.basename(
solid_file)
results['TopoSlopesX.FileName']['vals'][0][-1] = os.path.basename(slope_file_x)
results['TopoSlopesY.FileName']['vals'][0][-1] = os.path.basename(slope_file_y)
results['pfdist']['vals'][0][-1] = os.path.basename(slope_file_x)
results['pfdist']['vals'][1][-1] = os.path.basename(slope_file_y)
if evap_choice == 1:
results['pfdist']['vals'][2][-1] = os.path.basename(evap_file)
else:
results['pfdist']['vals'][2][0] = '#' + results['pfdist']['vals'][2][0]
# read slope file x into array
slope_x = pfio.pfread(slope_file_x)
nz0, ny0, nx0 = slope_x.shape
results['ComputationalGrid.NX']['vals'][0][-1] = str(nx0)
results['ComputationalGrid.NY']['vals'][0][-1] = str(ny0)
results['ComputationalGrid.NZ']['vals'][0][-1] = str(nz)
results['ComputationalGrid.DX']['vals'][0][-1] = str(dx)
results['ComputationalGrid.DY']['vals'][0][-1] = str(dy)
results['ComputationalGrid.DZ']['vals'][0][-1] = str(dz)
results['dzScale.nzListNumber']['vals'][0][-1] = str(nz)
for ni in range(nz):
results['Cell.' + str(ni) + '.dzScale.Value']['vals'][0][-1] = str(
dz_scales[ni])
dtype,
options=['COMPRESS=LZW'])
dataset.SetGeoTransform(geom)
dataset.SetProjection(prj)
for i in range(res_arr.shape[0]):
dataset.GetRasterBand(i + 1).WriteArray(res_arr[i, :, :])
dataset.GetRasterBand(i + 1).SetNoDataValue(ndata)
dataset.FlushCache()
in_file = sys.argv[1]
in_file_ext = os.path.splitext(in_file)[-1]
if in_file_ext == '.pfb':
in_arr = pfio.pfread(in_file)
nz, ny, nx = in_arr.shape
elif in_file_ext == '.txt':
in_arr = np.loadtxt(in_file, skiprows=1)
with open(in_file, 'r') as fi:
first_line = fi.readline().strip()
nx, ny, nz = [int(x) for x in first_line.split()]
in_arr = np.reshape(in_arr, (nz, ny, nx))
in_arr = in_arr[:, ::-1, :]
else:
print('does not support this file format..exit')
sys.exit()
#grid_coor_file = 'Grid_Centers_Short_Deg.format.txt'
grid_coor_file = gzip.GzipFile('grid_coor.npy.gz', "r")
grid_coor = np.load(grid_coor_file)
import numpy as np
import time
import pf_pytools.pf_fort_io as pf_fort_io
import pfio
input_pfb = '/home/arezaii/data/CONUS.5layer.pfclm.run4.out.clm_output.00001.C.pfb'
t3 = time.perf_counter()
fort_pfb_data = np.zeros((3342, 1888, 17), order='F')
pf_fort_io.pfb_read(fort_pfb_data, input_pfb)
fort_pfb_data = np.transpose(fort_pfb_data, (2, 1, 0))
t4 = time.perf_counter()
print(f'F load time {t4-t3:0.4f} seconds')
t1 = time.perf_counter()
c_pfb_data = pfio.pfread(input_pfb)
c_pfb_data = np.flip(c_pfb_data, axis=1)
t2 = time.perf_counter()
print(f'C load time {t2-t1:0.4f} seconds')
if np.testing.assert_array_equal(fort_pfb_data, c_pfb_data) is None:
print('arrays are equal')