def fill_gaps(tss, max_gap_to_fill=time_interval(hours=1)):
tss_filled = []
for ts in tss:
if ts is not None:
max_gap = int(max_gap_to_fill.total_seconds() / ts.interval.total_seconds())
if max_gap > 0:
tss_filled.append(interpolate_ts_nan(ts, max_gap=max_gap))
else:
tss_filled.append(ts)
else:
tss_filled.append(None)
return tss_filled
def fill_gaps_and_remove_ts_with_big_gaps(tss, max_gap):
""" Fill gaps and remove time series that cannot be filled
Returns
-------
list of time series
"""
ts_processed = []
for ts in tss:
# Fill gaps
ts_filled = interpolate_ts_nan(ts, max_gap=max_gap)
if np.any(np.isnan(ts_filled.data)):
# Drop the station
logger().warning("This station has too big a gap: %s",
ts_filled.props['name'])
else:
ts_processed.append(ts_filled)
return ts_processed
def main():
parser = create_arg_parser()
args = parser.parse_args()
monterey_fpath = args.monterey
pt_reyes_fpath = args.pt_reyes
hgrid_fpath = args.hgrid
fpath_out = args.outfile
#todo: hardwire
nnode = 83
tbuf = days(16)
# convert start time string input to datetime
sdate = datetime(*map(int, re.split('[^\d]', args.stime)))
if args.etime:
# convert start time string input to datetime
edate = datetime(*map(int, re.split('[^\d]', args.etime)))
bufend = edate + tbuf
else:
edate = None
bufend = None
# UTM positions of Point Reyes, Monterey, SF
pos_pr = np.array([502195.03, 4205445.47])
pos_mt = np.array([599422.84, 4051630.37])
pos_sf = np.array([547094.79, 4184499.42])
var_subtidal = np.array([0.938, 0.905, 0.969]) # pr, mt, sf
var_semi = np.array([0.554, 0.493, 0.580])
# Assume 45 degree from north-west to south-east
tangent = np.array([1, -1])
tangent = tangent / np.linalg.norm(tangent) # Normalize
# Rotate 90 cw to get normal vec
normal = np.array([tangent[1], -tangent[0]])
print "tangent:", tangent
print "normal:", normal
mt_rel = pos_mt - pos_pr
x_mt = np.dot(tangent, mt_rel) # In pr-mt direction
y_mt = np.dot(normal, mt_rel) # Normal to x-direction to the
# Grid
#todo: what is the difference between this and m = read_grid()??
s = create_schism_setup(hgrid_fpath)
ocean_boundary = s.mesh.boundaries[0] # First one is ocean
# Data
print "Reading Point Reyes..."
pt_reyes = read_ts(pt_reyes_fpath, start=sdate - tbuf, end=bufend, force_regular=True)
pt_reyes = interpolate_ts_nan(pt_reyes)
if np.any(np.isnan(pt_reyes.data)):
print pt_reyes.times[np.isnan(pt_reyes.data)]
ts_pr_subtidal, ts_pr_diurnal, ts_pr_semi, noise = separate_species(
pt_reyes)
del noise
print "Reading Monterey..."
monterey = read_ts(monterey_fpath, start=sdate - tbuf, end=bufend, force_regular=True)
monterey = interpolate_ts_nan(monterey)
if pt_reyes.interval != monterey.interval:
raise ValueError(
"Point Reyes and Monterey time step must be the same in gen_elev2D.py")
ts_mt_subtidal, ts_mt_diurnal, ts_mt_semi, noise = separate_species(
monterey)
del noise
dt = monterey.interval.seconds
print "Done Reading"
print "Interpolating Point Reyes"
# interpolate_ts(ts_pr_subtidal.window(sdate,edate),step)
ts_pr_subtidal = ts_pr_subtidal.window(sdate, edate)
ts_pr_diurnal = ts_pr_diurnal.window(sdate, edate) # interpolate_ts(,step)
# interpolate_ts(ts_pr_semi.window(sdate,edate),step)
ts_pr_semi = ts_pr_semi.window(sdate, edate)
print "Interpolating Monterey"
# interpolate_ts(ts_mt_subtidal.window(sdate,edate),step)
ts_mt_subtidal = ts_mt_subtidal.window(sdate, edate)
# interpolate_ts(ts_mt_diurnal.window(sdate,edate),step)
ts_mt_diurnal = ts_mt_diurnal.window(sdate, edate)
# interpolate_ts(ts_mt_semi.window(sdate,edate),step)
ts_mt_semi = ts_mt_semi.window(sdate, edate)
print "Creating writer" # requires dt be known for netcdf
if fpath_out.endswith("th"):
thwriter = BinaryTHWriter(fpath_out,nnode,None)
elif fpath_out.endswith("nc"):
thwriter = NetCDFTHWriter(fpath_out,nnode,sdate,dt)
else:
raise ValueError("File extension for output not recognized in file: {}".format(fpath_out))
# Grid
boundaries = s.mesh.nodes[ocean_boundary.nodes]
pos_rel = boundaries[:, :2] - pos_pr
# x, y in a new principal axes
x = np.dot(pos_rel, tangent.reshape((2, -1)))
y = np.dot(pos_rel, normal.reshape((2, -1)))
theta_x = x / x_mt
theta_x_comp = 1. - theta_x
theta_y = y / y_mt
theta_y_comp = 1. - theta_y
var_y = (theta_y_comp * var_semi[0] + theta_y * var_semi[1])
# adj_subtidal_mt = 0.08 # Adjustment in Monterey subtidal signal
# scaling_diurnal_mt = 0.95 # Scaling of Monterey diurnal signal (for K1/Q1)
# Used this up to v75
adj_subtidal_mt = 0. # Adjustment in Monterey subtidal signal
scaling_diurnal_mt = 1. # Scaling of Monterey diurnal signal (for K1/Q1)
# New trial for LSC2 with v75
adj_subtidal_mt = -0.07 # Adjustment in Monterey subtidal signal
scaling_diurnal_mt = 0.95 # Scaling of Monterey diurnal signal (for K1/Q1)
scaling_semidiurnal_mt = 1.03
adj_subtidal_mt = -0.14 # Adjustment in Monterey subtidal signal
scaling_diurnal_mt = 0.90 # Scaling of Monterey diurnal signal (for K1/Q1)
scaling_semidiurnal_mt = 1.07
adj_subtidal_mt = 0.10 # Adjustment in Monterey subtidal signal
scaling_diurnal_mt = 0.90 # Scaling of Monterey diurnal signal (for K1/Q1)
scaling_semidiurnal_mt = 1.03
adj_subtidal_mt = 0.10 # Adjustment in Monterey subtidal signal
scaling_diurnal_mt = 0.97 # Scaling of Monterey diurnal signal (for K1/Q1)
# Scaling of Point Reyes diurnal signal (for K1/Q1)
scaling_diurnal_pr = 0.97
scaling_semidiurnal_mt = 1.025 # Scaling at Monterey semi-diurnal signal
adj_subtidal_mt = 0.09 # Adjustment in Monterey subtidal signal
scaling_diurnal_mt = 0.94 # Scaling of Monterey diurnal signal (for K1/Q1)
# Scaling of Point Reyes diurnal signal (for K1/Q1)
scaling_diurnal_pr = 0.94
scaling_semidiurnal_mt = 1.0 # Scaling at Monterey semi-diurnal signal
slr = 0.0 # Sea level rise
if np.any(np.isnan(ts_pr_semi.data)):
print ts_pr_semi.times[np.isnan(ts_pr_semi.data)]
raise ValueError('')
# temp=np.zeros((len(ts_pr_semi),nnode))
# times[:]=[dt*i for i in xrange(len(ts_pr_semi))]
for i in xrange(len(ts_pr_semi)):
t = float(dt * i)
# semi-diurnal
# Scaling
pr = ts_pr_semi[i].value
mt = ts_mt_semi[i].value * scaling_semidiurnal_mt
if np.isnan(pr) or np.isnan(mt):
print pr
print mt
raise ValueError("One of values is numpy.nan.")
eta_pr_side = var_y / var_semi[0] * pr
eta_mt_side = var_y / var_semi[1] * mt
eta = eta_pr_side * theta_x_comp + eta_mt_side * theta_x
# diurnal
# Interpolate in x-direction only to get a better phase
pr = ts_pr_diurnal[i].value * scaling_diurnal_pr
mt = ts_mt_diurnal[i].value * scaling_diurnal_mt
if np.isnan(pr) or np.isnan(mt):
raise ValueError("One of values is numpy.nan.")
eta += pr * theta_x_comp + mt * theta_x
# Subtidal
# No phase change in x-direction. Simply interpolate in
# y-direction.
pr = ts_pr_subtidal[i].value + slr
mt = ts_mt_subtidal[i].value + adj_subtidal_mt + slr
if np.isnan(pr) or np.isnan(mt):
raise ValueError("One of values is numpy.nan.")
eta += pr * theta_y_comp + mt * theta_y
# write data to netCDF file
thwriter.write_step(i,t,eta)
# Delete class
del thwriter
def main():
""" Just a main function
"""
log = setup_logger()
# Read mesh
fpath = "hgrid.gr3"
mesh = SchismMeshIoFactory().get_reader('gr3').read(fpath)
nodes_as_point = [Point(node) for node in mesh.nodes]
# Create salts and nudging_factors
n_nodes = mesh.n_nodes()
nvrt = 23
nudging_factors = np.zeros((n_nodes, ))
# Read station db
fpath = "stations_utm.csv"
stations_db = StationDB(fpath)
# Read salt time series
# obs_dir = "../../selfe/BayDeltaSELFE/Data/CDEC_realtime/salt"
time_basis = datetime(2015, 8, 26)
time_start = time_basis
time_window = (time_start, datetime(2015, 9, 2))
# padding = timedelta(days=1)
# time_window_padded = (time_window[0] - padding, time_window[1] + padding)
fpath_csv_15min = '../Data/ec_15min.csv'
ec_15min = read_csv_from_dss(fpath_csv_15min)
fpath_csv_1hr = '../Data/ec_1hour.csv'
ec_1hr = read_csv_from_dss(fpath_csv_1hr)
ec_1hr.extend([
rts(ts.data[::4],
ts.times[0],
timedelta(hours=1),
props=deepcopy(ts.props)) for ts in ec_15min
])
# Read RKI to CDEC mapping
fpath_rki = '../Data/cdec_stas_nearterm.list'
rki_to_cdec = read_rki_to_cdec(fpath_rki)
for ts in ec_1hr:
if rki_to_cdec.get(ts.props['name']) is not None:
ts.props['name'] = rki_to_cdec[ts.props['name']]
ec_for_nudging = select_ec_in_stations_db(ec_1hr, stations_db)
# Cut out time windows
ec_for_nudging = [ts.window(*time_window) for ts in ec_for_nudging]
max_gap = 8 # 8 hours
ec_for_nudging = [
interpolate_ts_nan(ts, max_gap=max_gap) for ts in ec_for_nudging
]
ec_for_nudging = [
ts for ts in ec_for_nudging if not np.any(np.isnan(ts.data))
]
log.info("Total %d stations to nudge", len(ec_for_nudging))
if len(ec_for_nudging) < 1:
log.warning("No station to nudge...")
# Convert to PSU
log.info("Convert EC to PSU...")
list(map(ec_psu_25c, ec_for_nudging))
# Filtering, not doing it now
# ts_filt, filt = med_outliers(ts, level=6., range=[100., None])
# Collect masks nodes in the mesh for nudging of CDEC stations
log.info("Creating nudging masks...")
radius_of_nudging = 500.
radius_padding = 0.
nudging_pos = [ts.props['pos'] for ts in ec_for_nudging]
nudging_areas = [
Point(p).buffer(radius_of_nudging + radius_padding)
for p in nudging_pos
]
nudging_mask = np.full((n_nodes, ), -1., dtype=np.int32)
for node_idx, node in enumerate(nodes_as_point):
for nudging_idx, ball in enumerate(nudging_areas):
if ball.contains(node):
nudging_mask[node_idx] = nudging_idx
# Nudging factor
log.info("Creating nudging factors...")
station_nudging_factor = 1. / 86400.
station_nudging = np.zeros((n_nodes, ))
for node_idx, mask in enumerate(nudging_mask):
if mask >= 0:
center = Point(ec_for_nudging[mask].props['pos'])
dist = center.distance(nodes_as_point[node_idx])
station_nudging[node_idx] = np.max(1. - dist / radius_of_nudging,
0.) * station_nudging_factor
nudging_factors += station_nudging
log.info("Add ocean boundary...")
# Ocean nudging
# Nudging factor
ocean_nudging = generate_ocean_nudging_factors(mesh)
nudging_factors += ocean_nudging
# Add ocean nudging time series
ec_for_nudging.append(create_ocean_salt_ts(ec_for_nudging[0]))
# Add the ocean nudging mask
for node_idx in range(n_nodes):
if ocean_nudging[node_idx] > 0.:
nudging_mask[node_idx] = len(ec_for_nudging) - 1
# Write SAL_nudge.gr3
fpath_nudge_out = "SAL_nudge.gr3"
log.info("Creating %s", fpath_nudge_out)
SchismMeshIoFactory().get_writer('gr3').write(mesh=mesh,
fpath=fpath_nudge_out,
node_attr=nudging_factors)
# Write nu file
fpath_salt_nu = 'SAL_nu.in'
log.info("Creating %s", fpath_salt_nu)
if os.path.exists(fpath_salt_nu):
os.remove(fpath_salt_nu)
times = ec_for_nudging[0].times
print(times[0], times[-1])
times = [(t - time_basis).total_seconds() for t in times]
salt_background = 0.1
data = np.full((n_nodes, nvrt), salt_background)
for ts_idx, t in enumerate(times):
for node_idx, mask in enumerate(nudging_mask):
if mask >= 0:
data[node_idx, :] = ec_for_nudging[mask].data[ts_idx]
with open(fpath_salt_nu, 'ab') as f:
write_fortran_binary(f, np.array([t]))
for i in range(data.shape[0]):
write_fortran_binary(f, data[i])
# Quick copy and paste: Need to make these a function to reuse
# Write TEM_nudge.gr3
fpath_nudge_out = "TEM_nudge.gr3"
log.info("Creating %s", fpath_nudge_out)
SchismMeshIoFactory().get_writer('gr3').write(
mesh=mesh,
fpath=fpath_nudge_out,
node_attr=np.zeros_like(nudging_factors))
# Write nu file
fpath_temp_nu = 'TEM_nu.in'
log.info("Creating %s", fpath_temp_nu)
if os.path.exists(fpath_temp_nu):
os.remove(fpath_temp_nu)
# No temperature nudging here. 20 deg C everywhere
temp_background = 20.
data = np.full((n_nodes, nvrt), temp_background)
for ts_idx, t in enumerate(times):
# for node_idx, mask in enumerate(nudging_mask):
# if mask >= 0:
# data[node_idx, :] = ec_for_nudging[mask].data[ts_idx]
with open(fpath_temp_nu, 'ab') as f:
write_fortran_binary(f, np.array([t]))
for i in range(data.shape[0]):
write_fortran_binary(f, data[i])
