def test_reverse_up_left(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting,
end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point,
end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react(start_point=starting,
hit_point=int4d,
end_point=ending,
feature=intersection['feature'],
distance=distance,
angle=angle,
azimuth=difference['azimuth'],
reverse_azimuth=difference['reverse_azimuth'])
# Resulting latitude should be between the startpoint and the intersection point
assert final_point.latitude > int4d.latitude
assert final_point.latitude < starting.latitude
# Resulting longitude should be between the startpoint and the intersection point
assert final_point.longitude < int4d.longitude
assert final_point.longitude > starting.longitude
def test_reverse_half_distance_until_in_water(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting,
end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point,
end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react(start_point=starting,
hit_point=int4d,
end_point=ending,
feature=intersection['feature'],
distance=distance,
angle=angle,
azimuth=difference['azimuth'],
reverse_azimuth=difference['reverse_azimuth'],
reverse_distance=40000)
# Should be in water
assert s.intersect(start_point=final_point.point,
end_point=final_point.point) is None
def test_reverse_half_distance_until_in_water(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting, end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point, end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react( start_point = starting,
hit_point = int4d,
end_point = ending,
feature = intersection['feature'],
distance = distance,
angle = angle,
azimuth = difference['azimuth'],
reverse_azimuth = difference['reverse_azimuth'],
reverse_distance = 40000)
# Should be in water
assert s.intersect(start_point=final_point.point, end_point=final_point.point) is None
def test_reverse_up_left(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting, end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point, end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react( start_point = starting,
hit_point = int4d,
end_point = ending,
feature = intersection['feature'],
distance = distance,
angle = angle,
azimuth = difference['azimuth'],
reverse_azimuth = difference['reverse_azimuth'])
# Resulting latitude should be between the startpoint and the intersection point
assert final_point.latitude > int4d.latitude
assert final_point.latitude < starting.latitude
# Resulting longitude should be between the startpoint and the intersection point
assert final_point.longitude < int4d.longitude
assert final_point.longitude > starting.longitude
def test_reverse_distance_traveled(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting,
end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point,
end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react(start_point=starting,
hit_point=int4d,
end_point=ending,
feature=intersection['feature'],
distance=distance,
angle=angle,
azimuth=difference['azimuth'],
reverse_azimuth=difference['reverse_azimuth'],
reverse_distance=0.000001)
# Resulting point should be VERY close to the hit point.
assert abs(int4d.latitude - final_point.latitude) < 0.005
assert abs(int4d.longitude - final_point.longitude) < 0.005
def test_reverse_12_times_then_start_point(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting,
end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point,
end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react(start_point=starting,
hit_point=int4d,
end_point=ending,
feature=intersection['feature'],
distance=distance,
angle=angle,
azimuth=difference['azimuth'],
reverse_azimuth=difference['reverse_azimuth'],
reverse_distance=9999999999999999999999999999)
# Should be start location
assert final_point.longitude == starting.longitude
assert final_point.latitude == starting.latitude
assert final_point.depth == starting.depth
def test_reverse_left(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.1, longitude=-74.91, depth=0)
ending = Location4D(latitude=39.1, longitude=-74.85, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting, end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point, end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react( start_point = starting,
hit_point = int4d,
end_point = ending,
feature = intersection['feature'],
distance = distance,
angle = angle,
azimuth = difference['azimuth'],
reverse_azimuth = difference['reverse_azimuth'])
# Since we are on a stright horizonal line, the latitude will change only slightly
assert abs(final_point.latitude - starting.latitude) < 0.005
# Resulting longitude should be between the startpoint and the intersection point
assert final_point.longitude < int4d.longitude
assert final_point.longitude > starting.longitude
def test_reverse_10_times_then_start_point(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting, end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point, end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react( start_point = starting,
hit_point = int4d,
end_point = ending,
feature = intersection['feature'],
distance = distance,
angle = angle,
azimuth = difference['azimuth'],
reverse_azimuth = difference['reverse_azimuth'],
reverse_distance = 9999999999999999999999999999)
# Should be start location
assert final_point.longitude == starting.longitude
assert final_point.latitude == starting.latitude
assert final_point.depth == starting.depth
def test_reverse_left(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.1, longitude=-74.91, depth=0)
ending = Location4D(latitude=39.1, longitude=-74.85, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting,
end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point,
end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react(start_point=starting,
hit_point=int4d,
end_point=ending,
feature=intersection['feature'],
distance=distance,
angle=angle,
azimuth=difference['azimuth'],
reverse_azimuth=difference['reverse_azimuth'])
# Since we are on a stright horizonal line, the latitude will change only slightly
assert abs(final_point.latitude - starting.latitude) < 0.005
# Resulting longitude should be between the startpoint and the intersection point
assert final_point.longitude < int4d.longitude
assert final_point.longitude > starting.longitude
def test_reverse_distance_traveled(self):
s = Shoreline(type='reverse')
starting = Location4D(latitude=39.05, longitude=-75.34, depth=0)
ending = Location4D(latitude=38.96, longitude=-75.315, depth=0)
difference = AsaGreatCircle.great_distance(start_point=starting, end_point=ending)
angle = AsaMath.azimuth_to_math_angle(azimuth=difference['azimuth'])
distance = difference['distance']
intersection = s.intersect(start_point=starting.point, end_point=ending.point)
int4d = Location4D(point=intersection['point'])
final_point = s.react( start_point = starting,
hit_point = int4d,
end_point = ending,
feature = intersection['feature'],
distance = distance,
angle = angle,
azimuth = difference['azimuth'],
reverse_azimuth = difference['reverse_azimuth'],
reverse_distance = 0.000001)
# Resulting point should be VERY close to the hit point.
assert abs(int4d.latitude - final_point.latitude) < 0.005
assert abs(int4d.longitude - final_point.longitude) < 0.005
def get_info(self):
s = PTShoreline(path=self.path, feature_name=self.feature_name)
caps = s.get_feature_type_info()
self.path_type = unicode(type(s).__name__)
if caps is not None:
self.bbox = unicode(caps['LatLongBoundingBox'].wkt)
self.title = unicode(caps['Name'])
def test_reindexing(self):
p = Point(-73.745631, 40.336791)
p2 = Point(-78.745631, 44.336791)
p3 = Point(0, 0)
st = time.time()
s = Shoreline(point=p, spatialbuffer=2)
s.index(point=p2, spatialbuffer=2)
s.index(point=p3, spatialbuffer=2)
print "Reindexing Time: " + str(time.time() - st)
def test_reindexing(self):
p = Point(-73.745631, 40.336791)
p2 = Point(-78.745631, 44.336791)
p3 = Point(0, 0)
st = time.time()
s = Shoreline(point=p, spatialbuffer=2)
s.index(point=p2, spatialbuffer=2)
s.index(point=p3, spatialbuffer=2)
print "Reindexing Time: " + str(time.time() - st)
def test_large_shape_reindexing(self):
p = Point(-73.745631, 40.336791)
p2 = Point(-78.745631, 44.336791)
p3 = Point(0, 0)
st = time.time()
shore_path = os.path.join(self.shoreline_path, "alaska", "AK_Land_Basemap.shp")
s = Shoreline(file=shore_path, point=p, spatialbuffer=0.25)
s.index(point=p2, spatialbuffer=0.25)
s.index(point=p3, spatialbuffer=0.25)
print "Large Shoreline Reindexing Time: " + str(time.time() - st)
def test_multipart_shape_reindexing(self):
p = Point(-73.745631, 40.336791)
p2 = Point(-78.745631, 44.336791)
p3 = Point(0, 0)
st = time.time()
shore_path = os.path.join(self.shoreline_path, "westcoast", "New_Land_Clean.shp")
s = Shoreline(file=shore_path, point=p, spatialbuffer=1)
s.index(point=p2, spatialbuffer=0.25)
s.index(point=p3, spatialbuffer=0.25)
print "Multipart Shoreline Reindexing Time: " + str(time.time() - st)
def test_intersection_speed(self):
# Intersects on the west coast of NovaScotia
starting = Location4D(longitude=-66.1842219282406177, latitude=44.0141581697495852, depth=0).point
ending = Location4D(longitude=-66.1555195384399326, latitude=44.0387992322117370, depth=0).point
s = Shoreline(point=starting, spatialbuffer=1)
st = time.time()
intersection = s.intersect(start_point=starting, end_point=ending)['point']
print "Intersection Time: " + str(time.time() - st)
def test_multipart_shape_reindexing(self):
p = Point(-73.745631, 40.336791)
p2 = Point(-78.745631, 44.336791)
p3 = Point(0, 0)
st = time.time()
shore_path = os.path.join(self.shoreline_path, "westcoast",
"New_Land_Clean.shp")
s = Shoreline(file=shore_path, point=p, spatialbuffer=1)
s.index(point=p2, spatialbuffer=0.25)
s.index(point=p3, spatialbuffer=0.25)
print "Multipart Shoreline Reindexing Time: " + str(time.time() - st)
def test_large_shape_reindexing(self):
p = Point(-73.745631, 40.336791)
p2 = Point(-78.745631, 44.336791)
p3 = Point(0, 0)
st = time.time()
shore_path = os.path.join(self.shoreline_path, "alaska",
"AK_Land_Basemap.shp")
s = Shoreline(file=shore_path, point=p, spatialbuffer=0.25)
s.index(point=p2, spatialbuffer=0.25)
s.index(point=p3, spatialbuffer=0.25)
print "Large Shoreline Reindexing Time: " + str(time.time() - st)
def test_multipart_shape_intersection_speed(self):
# Intersects on the west coast of NovaScotia
starting = Location4D(longitude=-146.62, latitude=60.755, depth=0).point
ending = Location4D(longitude=-146.60, latitude=60.74, depth=0).point
shore_path = os.path.join(self.shoreline_path, "westcoast", "New_Land_Clean.shp")
s = Shoreline(file=shore_path, point=starting, spatialbuffer=1)
st = time.time()
intersection = s.intersect(start_point=starting, end_point=ending)['point']
print "Multipart Shoreline Intersection Time: " + str(time.time() - st)
def test_large_shape_intersection_speed(self):
# Intersects on the west coast of NovaScotia
starting = Location4D(longitude=-146.62, latitude=60.755, depth=0).point
ending = Location4D(longitude=-146.60, latitude=60.74, depth=0).point
shore_path = os.path.join(self.shoreline_path, "alaska", "AK_Land_Basemap.shp")
s = Shoreline(file=shore_path, point=starting, spatialbuffer=0.25)
st = time.time()
intersection = s.intersect(start_point=starting, end_point=ending)['point']
print "Large Shoreline Intersection Time: " + str(time.time() - st)
def test_water_start_land_end_intersection(self):
# Starts in the water and ends on land
s = Shoreline()
# -75, 39 is in the middle of the Delaware Bay
# -75, 39.5 is on land
# Intersection should be a Point starting somewhere around -75, 39.185 -> 39.195
starting = Location4D(latitude=39, longitude=-75, depth=0).point
ending = Location4D(latitude=39.5, longitude=-75, depth=0).point
intersection = Location4D(point=s.intersect(start_point=starting, end_point=ending)['point'])
assert -75 == intersection.longitude
assert intersection.latitude > 39.185
assert intersection.latitude < 39.195
def test_large_shape_intersection_speed(self):
# Intersects on the west coast of NovaScotia
starting = Location4D(longitude=-146.62, latitude=60.755,
depth=0).point
ending = Location4D(longitude=-146.60, latitude=60.74, depth=0).point
shore_path = os.path.join(self.shoreline_path, "alaska",
"AK_Land_Basemap.shp")
s = Shoreline(file=shore_path, point=starting, spatialbuffer=0.25)
st = time.time()
intersection = s.intersect(start_point=starting,
end_point=ending)['point']
print "Large Shoreline Intersection Time: " + str(time.time() - st)
def test_water_start_land_end_intersection(self):
# Starts in the water and ends on land
s = Shoreline()
# -75, 39 is in the middle of the Delaware Bay
# -75, 39.5 is on land
# Intersection should be a Point starting somewhere around -75, 39.185 -> 39.195
starting = Location4D(latitude=39, longitude=-75, depth=0).point
ending = Location4D(latitude=39.5, longitude=-75, depth=0).point
intersection = Location4D(
point=s.intersect(start_point=starting, end_point=ending)['point'])
assert -75 == intersection.longitude
assert intersection.latitude > 39.185
assert intersection.latitude < 39.195
def test_multipart_shape_intersection_speed(self):
# Intersects on the west coast of NovaScotia
starting = Location4D(longitude=-146.62, latitude=60.755,
depth=0).point
ending = Location4D(longitude=-146.60, latitude=60.74, depth=0).point
shore_path = os.path.join(self.shoreline_path, "westcoast",
"New_Land_Clean.shp")
s = Shoreline(file=shore_path, point=starting, spatialbuffer=1)
st = time.time()
intersection = s.intersect(start_point=starting,
end_point=ending)['point']
print "Multipart Shoreline Intersection Time: " + str(time.time() - st)
def test_intersection_speed(self):
# Intersects on the west coast of NovaScotia
starting = Location4D(longitude=-66.1842219282406177,
latitude=44.0141581697495852,
depth=0).point
ending = Location4D(longitude=-66.1555195384399326,
latitude=44.0387992322117370,
depth=0).point
s = Shoreline(point=starting, spatialbuffer=1)
st = time.time()
intersection = s.intersect(start_point=starting,
end_point=ending)['point']
print "Intersection Time: " + str(time.time() - st)
def test_water_start_water_end_jump_over_land_intersection(self):
# Starts on water and ends on water, but there is land inbetween
s = Shoreline()
# -75, 39 is in the middle of the Delaware Bay
# -74, 39 is in the Atlantic
# This jumps over a peninsula.
# Intersection should be the Point -74.96 -> -74.94, 39
#
starting = Location4D(latitude=39, longitude=-75, depth=0).point
ending = Location4D(latitude=39, longitude=-74, depth=0).point
intersection = Location4D(point=s.intersect(start_point=starting, end_point=ending)['point'])
assert 39 == intersection.latitude
assert intersection.longitude > -74.96
assert intersection.longitude < -74.94
def test_water_start_water_end_jump_over_land_intersection(self):
# Starts on water and ends on water, but there is land inbetween
s = Shoreline()
# -75, 39 is in the middle of the Delaware Bay
# -74, 39 is in the Atlantic
# This jumps over a peninsula.
# Intersection should be the Point -74.96 -> -74.94, 39
#
starting = Location4D(latitude=39, longitude=-75, depth=0).point
ending = Location4D(latitude=39, longitude=-74, depth=0).point
intersection = Location4D(
point=s.intersect(start_point=starting, end_point=ending)['point'])
assert 39 == intersection.latitude
assert intersection.longitude > -74.96
assert intersection.longitude < -74.94
def test_land_start_water_end_intersection(self):
# Starts on land and ends in the water
s = Shoreline()
# -75, 39.5 is on land
# -75, 39 is in the middle of the Delaware Bay
starting = Location4D(latitude=39.5, longitude=-75, depth=0).point
ending = Location4D(latitude=39, longitude=-75, depth=0).point
self.assertRaises(Exception,
s.intersect,
start_point=starting,
end_point=ending)
def test_land_start_land_end_intersection(self):
# Starts on land and ends on land
s = Shoreline()
# -75, 39.4 is on land
# -75, 39.5 is on land
starting = Location4D(latitude=39.4, longitude=-75, depth=0).point
ending = Location4D(latitude=39.5, longitude=-75, depth=0).point
self.assertRaises(Exception,
s.intersect,
start_point=starting,
end_point=ending)
def google_maps_coordinates(self, bbox=None):
marker_positions = []
if bbox:
s = PTShoreline(path=self.path, feature_name=self.feature_name)
geo_json = s.get_geoms_for_bounds(bbox)
geo = [asShape(g) for g in geo_json]
elif self.geometry:
geo = loads(self.geometry)
elif self.bbox:
geo = loads(self.bbox)
else:
return marker_positions
THRESHOLD=100000
# Google maps is y,x not x,y
if isinstance(geo, Point):
marker_positions.append((geo.coords[0][1], geo.coords[0][0]))
elif isinstance(geo, list):
total = 0
for g in geo:
if isinstance(g, Polygon):
total += len(g.exterior.coords)
if total > THRESHOLD:
return None
marker_positions.append([(pt[1], pt[0]) for pt in g.exterior.coords])
elif isinstance(g, MultiPolygon):
for subg in g:
total += len(subg.exterior.coords)
if total > THRESHOLD:
return None
marker_positions.append([(pt[1], pt[0]) for pt in subg.exterior.coords])
else:
for pt in geo.exterior.coords:
marker_positions.append((pt[1], pt[0]))
return marker_positions
def set_geometry(self, geo):
# If polygon is passed in, we need to trim it by the coastline
# so we don't start particles on land
if isinstance(geo, Polygon) and self._use_shoreline:
c = geo.centroid
b = geo.bounds
spatialbuffer = max(b[2] - b[0], b[3] - b[1])
shore_geoms = Shoreline(file=self.shoreline_path,
point=c,
spatialbuffer=spatialbuffer).geoms
if len(shore_geoms) > 0:
all_shore = cascaded_union(shore_geoms)
geo = geo.difference(all_shore)
self._geometry = geo
class ForceParticle(object):
from paegan.transport.shoreline import Shoreline
from paegan.transport.bathymetry import Bathymetry
def __str__(self):
return self.part.__str__()
def __init__(self, part, remotehydro, common_variables, timevar_pickle_path, times, start_time, models,
release_location_centroid, usebathy, useshore, usesurface,
get_data, n_run, read_lock, has_read_lock, read_count,
point_get, data_request_lock, has_data_request_lock, reverse_distance=None, bathy=None,
shoreline_path=None, cache=None, time_method=None):
"""
This is the task/class/object/job that forces an
individual particle and communicates with the
other particles and data controller for local
cache updates
"""
assert cache != None
self.cache_path = cache
self.bathy = bathy
self.common_variables = common_variables
self.localpath = self.cache_path
self.release_location_centroid = release_location_centroid
self.part = part
self.times = times
self.start_time = start_time
self.models = models
self.usebathy = usebathy
self.useshore = useshore
self.usesurface = usesurface
self.get_data = get_data
self.n_run = n_run
self.read_lock = read_lock
self.has_read_lock = has_read_lock
self.read_count = read_count
self.point_get = point_get
self.data_request_lock = data_request_lock
self.has_data_request_lock = has_data_request_lock
self.shoreline_path = shoreline_path
self.timevar_pickle_path = timevar_pickle_path
# Set common variable names
self.uname = common_variables.get("u", None)
self.vname = common_variables.get("v", None)
self.wname = common_variables.get("w", None)
self.temp_name = common_variables.get("temp", None)
self.salt_name = common_variables.get("salt", None)
self.xname = common_variables.get("x", None)
self.yname = common_variables.get("y", None)
self.zname = common_variables.get("z", None)
self.tname = common_variables.get("time", None)
self.reverse_distance = reverse_distance
if time_method is None:
time_method = 'interp'
self.time_method = time_method
def need_data(self, i):
"""
Method to test if cache contains the data that
the particle needs
"""
logger.debug("Checking cache for data availability at %s." % self.part.location.logstring())
try:
# Tell the DataController that we are going to be reading from the file
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
self.dataset.opennc()
# Test if the cache has the data we need
# If the point we request contains fill values,
# we need data
cached_lookup = self.dataset.get_values('domain', timeinds=[np.asarray([i])], point=self.part.location)
logger.debug("Type of result: %s" % type(cached_lookup))
logger.debug("Double mean of result: %s" % np.mean(np.mean(cached_lookup)))
logger.debug("Type of Double mean of result: %s" % type(np.mean(np.mean(cached_lookup))))
if type(np.mean(np.mean(cached_lookup))) == np.ma.core.MaskedConstant:
need = True
logger.debug("I NEED data. Got back: %s" % cached_lookup)
else:
need = False
logger.debug("I DO NOT NEED data")
except StandardError:
# If the time index doesnt even exist, we need
need = True
logger.debug("I NEED data (no time index exists in cache)")
finally:
self.dataset.closenc()
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
return need # return true if need data or false if dont
def linterp(self, setx, sety, x):
"""
Linear interp of model data values between time steps
"""
if math.isnan(sety[0]) or math.isnan(setx[0]):
return np.nan
#if math.isnan(sety[0]):
# sety[0] = 0.
#if math.isnan(sety[1]):
# sety[1] = 0.
return sety[0] + (x - setx[0]) * ( (sety[1]-sety[0]) / (setx[1]-setx[0]) )
def data_interp(self, i, timevar, currenttime):
"""
Method to streamline request for data from cache,
Uses linear interpolation bewtween timesteps to
get u,v,w,temp,salt
"""
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to release cache file so I can read from it...")
timer.sleep(4)
pass
if self.need_data(i+1):
# Acquire lock for asking for data
self.data_request_lock.acquire()
self.has_data_request_lock.value = os.getpid()
try:
# Do I still need data?
if self.need_data(i+1):
# Tell the DataController that we are going to be reading from the file
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
# Open netcdf file on disk from commondataset
self.dataset.opennc()
# Get the indices for the current particle location
indices = self.dataset.get_indices('u', timeinds=[np.asarray([i-1])], point=self.part.location )
self.dataset.closenc()
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
# Override the time
# get the current time index data
self.point_get.value = [indices[0] + 1, indices[-2], indices[-1]]
# Request that the data controller update the cache
self.get_data.value = True
# Wait until the data controller is done
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to update cache with the CURRENT time index")
timer.sleep(4)
pass
# get the next time index data
self.point_get.value = [indices[0] + 2, indices[-2], indices[-1]]
# Request that the data controller update the cache
self.get_data.value = True
# Wait until the data controller is done
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to update cache with the NEXT time index")
timer.sleep(4)
pass
except StandardError:
logger.warn("Particle failed to request data correctly")
raise
finally:
# Release lock for asking for data
self.has_data_request_lock.value = -1
self.data_request_lock.release()
# Tell the DataController that we are going to be reading from the file
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
try:
# Open netcdf file on disk from commondataset
self.dataset.opennc()
# Grab data at time index closest to particle location
u = [np.mean(np.mean(self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.part.location ))),
np.mean(np.mean(self.dataset.get_values('u', timeinds=[np.asarray([i+1])], point=self.part.location )))]
v = [np.mean(np.mean(self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.part.location ))),
np.mean(np.mean(self.dataset.get_values('v', timeinds=[np.asarray([i+1])], point=self.part.location )))]
# if there is vertical velocity inthe dataset, get it
if 'w' in self.dataset.nc.variables:
w = [np.mean(np.mean(self.dataset.get_values('w', timeinds=[np.asarray([i])], point=self.part.location ))),
np.mean(np.mean(self.dataset.get_values('w', timeinds=[np.asarray([i+1])], point=self.part.location )))]
else:
w = [0.0, 0.0]
# If there is salt and temp in the dataset, get it
if self.temp_name != None and self.salt_name != None:
temp = [np.mean(np.mean(self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.part.location ))),
np.mean(np.mean(self.dataset.get_values('temp', timeinds=[np.asarray([i+1])], point=self.part.location )))]
salt = [np.mean(np.mean(self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.part.location ))),
np.mean(np.mean(self.dataset.get_values('salt', timeinds=[np.asarray([i+1])], point=self.part.location )))]
# Check for nans that occur in the ocean (happens because
# of model and coastline resolution mismatches)
if np.isnan(u).any() or np.isnan(v).any() or np.isnan(w).any():
# Take the mean of the closest 4 points
# If this includes nan which it will, result is nan
uarray1 = self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.part.location, num=2)
varray1 = self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.part.location, num=2)
uarray2 = self.dataset.get_values('u', timeinds=[np.asarray([i+1])], point=self.part.location, num=2)
varray2 = self.dataset.get_values('v', timeinds=[np.asarray([i+1])], point=self.part.location, num=2)
if 'w' in self.dataset.nc.variables:
warray1 = self.dataset.get_values('w', timeinds=[np.asarray([i])], point=self.part.location, num=2)
warray2 = self.dataset.get_values('w', timeinds=[np.asarray([i+1])], point=self.part.location, num=2)
w = [warray1.mean(), warray2.mean()]
else:
w = [0.0, 0.0]
if self.temp_name != None and self.salt_name != None:
temparray1 = self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.part.location, num=2)
saltarray1 = self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.part.location, num=2)
temparray2 = self.dataset.get_values('temp', timeinds=[np.asarray([i+1])], point=self.part.location, num=2)
saltarray2 = self.dataset.get_values('salt', timeinds=[np.asarray([i+1])], point=self.part.location, num=2)
temp = [temparray1.mean(), temparray2.mean()]
salt = [saltarray1.mean(), saltarray2.mean()]
u = [uarray1.mean(), uarray2.mean()]
v = [varray1.mean(), varray2.mean()]
# Linear interp of data between timesteps
currenttime = date2num(currenttime)
timevar = timevar.datenum
u = self.linterp(timevar[i:i+2], u, currenttime)
v = self.linterp(timevar[i:i+2], v, currenttime)
w = self.linterp(timevar[i:i+2], w, currenttime)
if self.temp_name != None and self.salt_name != None:
temp = self.linterp(timevar[i:i+2], temp, currenttime)
salt = self.linterp(timevar[i:i+2], salt, currenttime)
if self.temp_name is None:
temp = np.nan
if self.salt_name is None:
salt = np.nan
#logger.info(self.dataset.get_xyind_from_point('u', self.part.location, num=1))
except StandardError:
logger.error("Error in data_interp method on ForceParticle")
raise
finally:
self.dataset.closenc()
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
return u, v, w, temp, salt
def data_nearest(self, i, currenttime):
"""
Method to streamline request for data from cache,
Uses nearest time to get u,v,w,temp,salt
"""
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to release cache file so I can read from it...")
timer.sleep(4)
pass
if self.need_data(i):
# Acquire lock for asking for data
self.data_request_lock.acquire()
self.has_data_request_lock.value = os.getpid()
try:
if self.need_data(i):
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
# Open netcdf file on disk from commondataset
self.dataset.opennc()
# Get the indices for the current particle location
indices = self.dataset.get_indices('u', timeinds=[np.asarray([i-1])], point=self.part.location )
self.dataset.closenc()
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
# Override the time
self.point_get.value = [indices[0]+1, indices[-2], indices[-1]]
# Request that the data controller update the cache
# DATA CONTOLLER STARTS
self.get_data.value = True
# Wait until the data controller is done
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to update cache...")
timer.sleep(4)
pass
except StandardError:
raise
finally:
self.has_data_request_lock.value = -1
self.data_request_lock.release()
# Tell the DataController that we are going to be reading from the file
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
try:
# Open netcdf file on disk from commondataset
self.dataset.opennc()
# Grab data at time index closest to particle location
u = np.mean(np.mean(self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.part.location )))
v = np.mean(np.mean(self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.part.location )))
# if there is vertical velocity inthe dataset, get it
if 'w' in self.dataset.nc.variables:
w = np.mean(np.mean(self.dataset.get_values('w', timeindsf=[np.asarray([i])], point=self.part.location )))
else:
w = 0.0
# If there is salt and temp in the dataset, get it
if self.temp_name != None and self.salt_name != None:
temp = np.mean(np.mean(self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.part.location )))
salt = np.mean(np.mean(self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.part.location )))
# Check for nans that occur in the ocean (happens because
# of model and coastline resolution mismatches)
if np.isnan(u).any() or np.isnan(v).any() or np.isnan(w).any():
# Take the mean of the closest 4 points
# If this includes nan which it will, result is nan
uarray1 = self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.part.location, num=2)
varray1 = self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.part.location, num=2)
if 'w' in self.dataset.nc.variables:
warray1 = self.dataset.get_values('w', timeinds=[np.asarray([i])], point=self.part.location, num=2)
w = warray1.mean()
else:
w = 0.0
if self.temp_name != None and self.salt_name != None:
temparray1 = self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.part.location, num=2)
saltarray1 = self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.part.location, num=2)
temp = temparray1.mean()
salt = saltarray1.mean()
u = uarray1.mean()
v = varray1.mean()
if self.temp_name is None:
temp = np.nan
if self.salt_name is None:
salt = np.nan
#logger.info(self.dataset.get_xyind_from_point('u', self.part.location, num=1))
except StandardError:
logger.error("Error in data_nearest on ForceParticle")
raise
finally:
self.dataset.closenc()
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
return u, v, w, temp, salt
def __call__(self, proc, active):
self.active = active
if self.usebathy == True:
self._bathymetry = Bathymetry(file=self.bathy)
self._shoreline = None
if self.useshore == True:
self._shoreline = Shoreline(file=self.shoreline_path, point=self.release_location_centroid, spatialbuffer=0.25)
# Make sure we are not starting on land. Raises exception if we are.
self._shoreline.intersect(start_point=self.release_location_centroid, end_point=self.release_location_centroid)
self.proc = proc
part = self.part
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to start...")
timer.sleep(10)
pass
# Initialize commondataset of local cache, then
# close the related netcdf file
try:
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
self.dataset = CommonDataset.open(self.localpath)
self.dataset.closenc()
except StandardError:
logger.warn("No cache file: %s. Particle exiting" % self.localpath)
raise
finally:
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
# Calculate datetime at every timestep
modelTimestep, newtimes = AsaTransport.get_time_objects_from_model_timesteps(self.times, start=self.start_time)
# Load Timevar from pickle serialization
f = open(self.timevar_pickle_path,"rb")
timevar = pickle.load(f)
f.close()
if self.time_method == 'interp':
time_indexs = timevar.nearest_index(newtimes, select='before')
elif self.time_method == 'nearest':
time_indexs = timevar.nearest_index(newtimes)
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
try:
assert len(newtimes) == len(time_indexs)
except AssertionError:
logger.error("Time indexes are messed up. Need to have equal datetime and time indexes")
raise
# loop over timesteps
# We don't loop over the last time_index because
# we need to query in the time_index and set the particle's
# location as the 'newtime' object.
for loop_i, i in enumerate(time_indexs[0:-1]):
if self.active.value == False:
raise ValueError("Particle exiting due to Failure.")
newloc = None
# if need a time that is outside of what we have
#if self.active.value == True:
# while self.get_data.value == True:
# logger.info("Waiting for DataController to get out...")
# timer.sleep(4)
# pass
# Get the variable data required by the models
if self.time_method == 'nearest':
u, v, w, temp, salt = self.data_nearest(i, newtimes[loop_i])
elif self.time_method == 'interp':
u, v, w, temp, salt = self.data_interp(i, timevar, newtimes[loop_i])
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
#logger.info("U: %.4f, V: %.4f, W: %.4f" % (u,v,w))
#logger.info("Temp: %.4f, Salt: %.4f" % (temp,salt))
# Get the bathy value at the particles location
if self.usebathy == True:
bathymetry_value = self._bathymetry.get_depth(part.location)
else:
bathymetry_value = -999999999999999
# Age the particle by the modelTimestep (seconds)
# 'Age' meaning the amount of time it has been forced.
part.age(seconds=modelTimestep[loop_i])
# loop over models - sort these in the order you want them to run
for model in self.models:
movement = model.move(part, u, v, w, modelTimestep[loop_i], temperature=temp, salinity=salt, bathymetry_value=bathymetry_value)
newloc = Location4D(latitude=movement['latitude'], longitude=movement['longitude'], depth=movement['depth'], time=newtimes[loop_i+1])
logger.debug("%s - moved %.3f meters (horizontally) and %.3f meters (vertically) by %s with data from %s" % (part.logstring(), movement['distance'], movement['vertical_distance'], model.__class__.__name__, newtimes[loop_i].isoformat()))
if newloc:
self.boundary_interaction(particle=part, starting=part.location, ending=newloc,
distance=movement['distance'], angle=movement['angle'],
azimuth=movement['azimuth'], reverse_azimuth=movement['reverse_azimuth'],
vertical_distance=movement['vertical_distance'], vertical_angle=movement['vertical_angle'])
logger.debug("%s - was forced by %s and is now at %s" % (part.logstring(), model.__class__.__name__, part.location.logstring()))
part.note = part.outputstring()
# Each timestep, save the particles status and environmental variables.
# This keep fields such as temp, salt, halted, settled, and dead matched up with the number of timesteps
part.save()
# We won't pull data for the last entry in locations, but we need to populate it with fill data.
part.fill_environment_gap()
if self.usebathy == True:
self._bathymetry.close()
if self.useshore == True:
self._shoreline.close()
return part
def boundary_interaction(self, **kwargs):
"""
Returns a list of Location4D objects
"""
particle = kwargs.pop('particle')
starting = kwargs.pop('starting')
ending = kwargs.pop('ending')
# shoreline
if self.useshore:
intersection_point = self._shoreline.intersect(start_point=starting.point, end_point=ending.point)
if intersection_point:
# Set the intersection point.
hitpoint = Location4D(point=intersection_point['point'], time=starting.time + (ending.time - starting.time))
particle.location = hitpoint
# This relies on the shoreline to put the particle in water and not on shore.
resulting_point = self._shoreline.react(start_point=starting,
end_point=ending,
hit_point=hitpoint,
reverse_distance=self.reverse_distance,
feature=intersection_point['feature'],
distance=kwargs.get('distance'),
angle=kwargs.get('angle'),
azimuth=kwargs.get('azimuth'),
reverse_azimuth=kwargs.get('reverse_azimuth'))
ending.latitude = resulting_point.latitude
ending.longitude = resulting_point.longitude
ending.depth = resulting_point.depth
logger.debug("%s - hit the shoreline at %s. Setting location to %s." % (particle.logstring(), hitpoint.logstring(), ending.logstring()))
# bathymetry
if self.usebathy:
if not particle.settled:
bintersect = self._bathymetry.intersect(start_point=starting, end_point=ending)
if bintersect:
pt = self._bathymetry.react(type='reverse', start_point=starting, end_point=ending)
logger.debug("%s - hit the bottom at %s. Setting location to %s." % (particle.logstring(), ending.logstring(), pt.logstring()))
ending.latitude = pt.latitude
ending.longitude = pt.longitude
ending.depth = pt.depth
# sea-surface
if self.usesurface:
if ending.depth > 0:
#logger.debug("%s - rose out of the water. Setting depth to 0." % particle.logstring())
ending.depth = 0
particle.location = ending
return
def run(self):
self.load_initial_dataset()
redis_connection = None
if self.redis_url is not None and self.redis_results_channel is not None:
import redis
redis_connection = redis.from_url(self.redis_url)
# Setup shoreline
self._shoreline = None
if self.useshore is True:
self._shoreline = Shoreline(path=self.shoreline_path, feature_name=self.shoreline_feature, point=self.release_location_centroid, spatialbuffer=self.shoreline_index_buffer)
# Make sure we are not starting on land. Raises exception if we are.
self._shoreline.intersect(start_point=self.release_location_centroid, end_point=self.release_location_centroid)
# Setup Bathymetry
if self.usebathy is True:
try:
self._bathymetry = Bathymetry(file=self.bathy_path)
except Exception:
logger.exception("Could not load Bathymetry file: %s, using no Bathymetry for this run!" % self.bathy_path)
self.usebathy = False
# Calculate datetime at every timestep
modelTimestep, newtimes = AsaTransport.get_time_objects_from_model_timesteps(self.times, start=self.start_time)
if self.time_method == 'interp':
time_indexs = self.timevar.nearest_index(newtimes, select='before')
elif self.time_method == 'nearest':
time_indexs = self.timevar.nearest_index(newtimes)
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
try:
assert len(newtimes) == len(time_indexs)
except AssertionError:
logger.exception("Time indexes are messed up. Need to have equal datetime and time indexes")
raise
# Keep track of how much time we spend in each area.
tot_boundary_time = 0.
tot_model_time = {}
tot_read_data = 0.
for m in self.models:
tot_model_time[m.name] = 0.
# Set the base conditions
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(self.redis_results_channel, json.dumps(self.particle.timestep_dump()))
# loop over timesteps
# We don't loop over the last time_index because
# we need to query in the time_index and set the particle's
# location as the 'newtime' object.
for loop_i, i in enumerate(time_indexs[0:-1]):
if self.active and self.active.value is False:
raise ValueError("Particle exiting due to Failure.")
newloc = None
st = time.clock()
# Get the variable data required by the models
if self.time_method == 'nearest':
u, v, w, temp, salt = self.get_nearest_data(i)
elif self.time_method == 'interp':
u, v, w, temp, salt = self.get_linterp_data(i, newtimes[loop_i])
else:
logger.warn("Method for computing u,v,w,temp,salt is unknown. Only 'nearest' and 'interp' are supported.")
tot_read_data += (time.clock() - st)
# Get the bathy value at the particles location
if self.usebathy is True:
bathymetry_value = self._bathymetry.get_depth(self.particle.location)
else:
bathymetry_value = -999999999999999
# Age the particle by the modelTimestep (seconds)
# 'Age' meaning the amount of time it has been forced.
self.particle.age(seconds=modelTimestep[loop_i])
# loop over models - sort these in the order you want them to run
for model in self.models:
st = time.clock()
movement = model.move(self.particle, u, v, w, modelTimestep[loop_i], temperature=temp, salinity=salt, bathymetry_value=bathymetry_value)
newloc = Location4D(latitude=movement['latitude'], longitude=movement['longitude'], depth=movement['depth'], time=newtimes[loop_i+1])
tot_model_time[m.name] += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - moved %.3f meters (horizontally) and %.3f meters (vertically) by %s with data from %s" % (self.particle.logstring(), movement['distance'], movement['vertical_distance'], model.__class__.__name__, newtimes[loop_i].isoformat()))
if newloc:
st = time.clock()
self.boundary_interaction(particle=self.particle, starting=self.particle.location, ending=newloc,
distance=movement['distance'], angle=movement['angle'],
azimuth=movement['azimuth'], reverse_azimuth=movement['reverse_azimuth'],
vertical_distance=movement['vertical_distance'], vertical_angle=movement['vertical_angle'])
tot_boundary_time += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - was forced by %s and is now at %s" % (self.particle.logstring(), model.__class__.__name__, self.particle.location.logstring()))
self.particle.note = self.particle.outputstring()
# Each timestep, save the particles status and environmental variables.
# This keep fields such as temp, salt, halted, settled, and dead matched up with the number of timesteps
self.particle.save()
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(self.redis_results_channel, json.dumps(self.particle.timestep_dump()))
self.dataset.closenc()
# We won't pull data for the last entry in locations, but we need to populate it with fill data.
self.particle.fill_gap()
if self.usebathy is True:
self._bathymetry.close()
if self.useshore is True:
self._shoreline.close()
logger.info(textwrap.dedent('''Particle %i Stats:
Data read: %f seconds
Model forcing: %s seconds
Boundary intersection: %f seconds''' % (self.particle.uid, tot_read_data, { s: '{:g} seconds'.format(f) for s, f in list(tot_model_time.items()) }, tot_boundary_time)))
return self.particle
class BaseForcer(object):
def __init__(self, hydrodataset, **kwargs):
"""
part, common_variables, timevar, times, start_time, models,
release_location_centroid, usebathy, useshore, usesurface,
get_data, n_run, read_lock, has_read_lock, read_count,
point_get, data_request_lock, has_data_request_lock, reverse_distance=None, bathy=None,
shoreline_path=None, shoreline_feature=None, time_method=None, caching=None, redis_url=None, redis_results_channel=None, shoreline_index_buffer=None):
This is the task/class/object/job that forces an
individual particle and communicates with the
other particles and data controller for local
cache updates
"""
assert hydrodataset is not None
# Common parameters
self.hydrodataset = hydrodataset
self.bathy_path = kwargs.get("bathy_path")
self.release_location_centroid = kwargs.get("release_location_centroid")
self.particle = kwargs.get("particle")
self.times = kwargs.get("times")
self.timevar = kwargs.get("timevar", None)
self.start_time = kwargs.get("start_time")
self.models = kwargs.get("models", [])
self.usebathy = kwargs.get("usebathy", False)
self.useshore = kwargs.get("useshore", False)
self.usesurface = kwargs.get("usesurface", True)
self.shoreline_path = kwargs.get("shoreline_path")
self.shoreline_feature = kwargs.get("shoreline_feature", None)
self.shoreline_index_buffer = kwargs.get("shoreline_index_buffer", 0.1)
self.time_method = kwargs.get("time_method", "nearest")
self.reverse_distance = kwargs.get("reverse_distance", 500)
# Redis for results
self.redis_url = kwargs.get("redis_url", None)
self.redis_results_channel = kwargs.get("redis_results_channel", None)
# Set common variable names
self.common_variables = kwargs.get("common_variables")
self.uname = self.common_variables.get("u", None)
self.vname = self.common_variables.get("v", None)
self.wname = self.common_variables.get("w", None)
self.temp_name = self.common_variables.get("temp", None)
self.salt_name = self.common_variables.get("salt", None)
self.xname = self.common_variables.get("x", None)
self.yname = self.common_variables.get("y", None)
self.zname = self.common_variables.get("z", None)
self.tname = self.common_variables.get("time", None)
self.active = None
def load_initial_dataset(self):
"""
Initialize self.dataset, then close it
A cacher will have to wrap this in locks, while a straight runner will not.
"""
try:
self.dataset = CommonDataset.open(self.hydrodataset)
if self.timevar is None:
self.timevar = self.dataset.gettimevar(self.common_variables.get("u"))
except Exception:
logger.warn("No source dataset: %s. Particle exiting" % self.hydrodataset)
raise
def boundary_interaction(self, **kwargs):
"""
Returns a list of Location4D objects
"""
particle = kwargs.pop('particle')
starting = kwargs.pop('starting')
ending = kwargs.pop('ending')
# shoreline
if self.useshore:
intersection_point = self._shoreline.intersect(start_point=starting.point, end_point=ending.point)
if intersection_point is not None:
# Set the intersection point.
hitpoint = Location4D(point=intersection_point['point'], time=starting.time + (ending.time - starting.time))
particle.location = hitpoint
# This relies on the shoreline to put the particle in water and not on shore.
resulting_point = self._shoreline.react(start_point=starting,
end_point=ending,
hit_point=hitpoint,
reverse_distance=self.reverse_distance,
feature=intersection_point['feature'],
distance=kwargs.get('distance'),
angle=kwargs.get('angle'),
azimuth=kwargs.get('azimuth'),
reverse_azimuth=kwargs.get('reverse_azimuth'))
ending.latitude = resulting_point.latitude
ending.longitude = resulting_point.longitude
ending.depth = resulting_point.depth
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - hit the shoreline at %s. Setting location to %s." % (particle.logstring(), hitpoint.logstring(), ending.logstring()))
# bathymetry
if self.usebathy:
if not particle.settled:
bintersect = self._bathymetry.intersect(start_point=starting, end_point=ending)
if bintersect:
pt = self._bathymetry.react(type='reverse', start_point=starting, end_point=ending)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - hit the bottom at %s. Setting location to %s." % (particle.logstring(), ending.logstring(), pt.logstring()))
ending.latitude = pt.latitude
ending.longitude = pt.longitude
ending.depth = pt.depth
# sea-surface
if self.usesurface:
if ending.depth > 0:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - rose out of the water. Setting depth to 0." % particle.logstring())
ending.depth = 0
particle.location = ending
def get_nearest_data(self, i):
""" Note: self.dataset.opennc() must be called before calling this function.
This is because the caching forcer must close it everytime, while a non caching
forcer can leave the dataset open.
"""
try:
# Grab data at time index closest to particle location
u = np.mean(np.mean(self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.particle.location )))
v = np.mean(np.mean(self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.particle.location )))
# if there is vertical velocity inthe dataset, get it
if 'w' in self.dataset.nc.variables:
w = np.mean(np.mean(self.dataset.get_values('w', timeindsf=[np.asarray([i])], point=self.particle.location )))
else:
w = 0.0
# If there is salt and temp in the dataset, get it
if self.temp_name is not None and self.salt_name is not None:
temp = np.mean(np.mean(self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.particle.location )))
salt = np.mean(np.mean(self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.particle.location )))
# Check for nans that occur in the ocean (happens because
# of model and coastline resolution mismatches)
if np.isnan(u).any() or np.isnan(v).any() or np.isnan(w).any():
# Take the mean of the closest 4 points
# If this includes nan which it will, result is nan
uarray1 = self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
varray1 = self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
if 'w' in self.dataset.nc.variables:
warray1 = self.dataset.get_values('w', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
w = warray1.mean()
else:
w = 0.0
if self.temp_name is not None and self.salt_name is not None:
temparray1 = self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
saltarray1 = self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
temp = temparray1.mean()
salt = saltarray1.mean()
u = uarray1.mean()
v = varray1.mean()
if self.temp_name is None:
temp = np.nan
if self.salt_name is None:
salt = np.nan
except Exception:
logger.exception("Could not retrieve data.")
raise
return u, v, w, temp, salt
def get_linterp_data(self, i, currenttime):
""" Note: self.dataset.opennc() must be called before calling this function.
This is because the caching forcer must close it everytime, while a non caching
forcer can leave the dataset open.
"""
try:
# Grab data at time index closest to particle location
u = [np.mean(np.mean(self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.particle.location ))),
np.mean(np.mean(self.dataset.get_values('u', timeinds=[np.asarray([i+1])], point=self.particle.location )))]
v = [np.mean(np.mean(self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.particle.location ))),
np.mean(np.mean(self.dataset.get_values('v', timeinds=[np.asarray([i+1])], point=self.particle.location )))]
# if there is vertical velocity inthe dataset, get it
if 'w' in self.dataset.nc.variables:
w = [np.mean(np.mean(self.dataset.get_values('w', timeinds=[np.asarray([i])], point=self.particle.location ))),
np.mean(np.mean(self.dataset.get_values('w', timeinds=[np.asarray([i+1])], point=self.particle.location )))]
else:
w = [0.0, 0.0]
# If there is salt and temp in the dataset, get it
if self.temp_name is not None and self.salt_name is not None:
temp = [np.mean(np.mean(self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.particle.location ))),
np.mean(np.mean(self.dataset.get_values('temp', timeinds=[np.asarray([i+1])], point=self.particle.location )))]
salt = [np.mean(np.mean(self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.particle.location ))),
np.mean(np.mean(self.dataset.get_values('salt', timeinds=[np.asarray([i+1])], point=self.particle.location )))]
# Check for nans that occur in the ocean (happens because
# of model and coastline resolution mismatches)
if np.isnan(u).any() or np.isnan(v).any() or np.isnan(w).any():
# Take the mean of the closest 4 points
# If this includes nan which it will, result is nan
uarray1 = self.dataset.get_values('u', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
varray1 = self.dataset.get_values('v', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
uarray2 = self.dataset.get_values('u', timeinds=[np.asarray([i+1])], point=self.particle.location, num=2)
varray2 = self.dataset.get_values('v', timeinds=[np.asarray([i+1])], point=self.particle.location, num=2)
if 'w' in self.dataset.nc.variables:
warray1 = self.dataset.get_values('w', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
warray2 = self.dataset.get_values('w', timeinds=[np.asarray([i+1])], point=self.particle.location, num=2)
w = [warray1.mean(), warray2.mean()]
else:
w = [0.0, 0.0]
if self.temp_name is not None and self.salt_name is not None:
temparray1 = self.dataset.get_values('temp', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
saltarray1 = self.dataset.get_values('salt', timeinds=[np.asarray([i])], point=self.particle.location, num=2)
temparray2 = self.dataset.get_values('temp', timeinds=[np.asarray([i+1])], point=self.particle.location, num=2)
saltarray2 = self.dataset.get_values('salt', timeinds=[np.asarray([i+1])], point=self.particle.location, num=2)
temp = [temparray1.mean(), temparray2.mean()]
salt = [saltarray1.mean(), saltarray2.mean()]
u = [uarray1.mean(), uarray2.mean()]
v = [varray1.mean(), varray2.mean()]
# Linear interp of data between timesteps
currenttime = date2num(currenttime)
timevar = self.timevar.datenum
u = self.linterp(timevar[i:i+2], u, currenttime)
v = self.linterp(timevar[i:i+2], v, currenttime)
w = self.linterp(timevar[i:i+2], w, currenttime)
if self.temp_name is not None and self.salt_name is not None:
temp = self.linterp(timevar[i:i+2], temp, currenttime)
salt = self.linterp(timevar[i:i+2], salt, currenttime)
if self.temp_name is None:
temp = np.nan
if self.salt_name is None:
salt = np.nan
except Exception:
logger.exception("Could not retrieve data.")
raise
return u, v, w, temp, salt
def linterp(self, setx, sety, x):
"""
Linear interp of model data values between time steps
"""
if math.isnan(sety[0]) or math.isnan(setx[0]):
return np.nan
return sety[0] + (x - setx[0]) * ( (sety[1]-sety[0]) / (setx[1]-setx[0]) )
def run(self):
self.load_initial_dataset()
redis_connection = None
if self.redis_url is not None and self.redis_results_channel is not None:
import redis
redis_connection = redis.from_url(self.redis_url)
# Setup shoreline
self._shoreline = None
if self.useshore is True:
self._shoreline = Shoreline(path=self.shoreline_path, feature_name=self.shoreline_feature, point=self.release_location_centroid, spatialbuffer=self.shoreline_index_buffer)
# Make sure we are not starting on land. Raises exception if we are.
self._shoreline.intersect(start_point=self.release_location_centroid, end_point=self.release_location_centroid)
# Setup Bathymetry
if self.usebathy is True:
try:
self._bathymetry = Bathymetry(file=self.bathy_path)
except Exception:
logger.exception("Could not load Bathymetry file: %s, using no Bathymetry for this run!" % self.bathy_path)
self.usebathy = False
# Calculate datetime at every timestep
modelTimestep, newtimes = AsaTransport.get_time_objects_from_model_timesteps(self.times, start=self.start_time)
if self.time_method == 'interp':
time_indexs = self.timevar.nearest_index(newtimes, select='before')
elif self.time_method == 'nearest':
time_indexs = self.timevar.nearest_index(newtimes)
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
try:
assert len(newtimes) == len(time_indexs)
except AssertionError:
logger.exception("Time indexes are messed up. Need to have equal datetime and time indexes")
raise
# Keep track of how much time we spend in each area.
tot_boundary_time = 0.
tot_model_time = {}
tot_read_data = 0.
for m in self.models:
tot_model_time[m.name] = 0.
# Set the base conditions
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(self.redis_results_channel, json.dumps(self.particle.timestep_dump()))
# loop over timesteps
# We don't loop over the last time_index because
# we need to query in the time_index and set the particle's
# location as the 'newtime' object.
for loop_i, i in enumerate(time_indexs[0:-1]):
if self.active and self.active.value is False:
raise ValueError("Particle exiting due to Failure.")
newloc = None
st = time.clock()
# Get the variable data required by the models
if self.time_method == 'nearest':
u, v, w, temp, salt = self.get_nearest_data(i)
elif self.time_method == 'interp':
u, v, w, temp, salt = self.get_linterp_data(i, newtimes[loop_i])
else:
logger.warn("Method for computing u,v,w,temp,salt is unknown. Only 'nearest' and 'interp' are supported.")
tot_read_data += (time.clock() - st)
# Get the bathy value at the particles location
if self.usebathy is True:
bathymetry_value = self._bathymetry.get_depth(self.particle.location)
else:
bathymetry_value = -999999999999999
# Age the particle by the modelTimestep (seconds)
# 'Age' meaning the amount of time it has been forced.
self.particle.age(seconds=modelTimestep[loop_i])
# loop over models - sort these in the order you want them to run
for model in self.models:
st = time.clock()
movement = model.move(self.particle, u, v, w, modelTimestep[loop_i], temperature=temp, salinity=salt, bathymetry_value=bathymetry_value)
newloc = Location4D(latitude=movement['latitude'], longitude=movement['longitude'], depth=movement['depth'], time=newtimes[loop_i+1])
tot_model_time[m.name] += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - moved %.3f meters (horizontally) and %.3f meters (vertically) by %s with data from %s" % (self.particle.logstring(), movement['distance'], movement['vertical_distance'], model.__class__.__name__, newtimes[loop_i].isoformat()))
if newloc:
st = time.clock()
self.boundary_interaction(particle=self.particle, starting=self.particle.location, ending=newloc,
distance=movement['distance'], angle=movement['angle'],
azimuth=movement['azimuth'], reverse_azimuth=movement['reverse_azimuth'],
vertical_distance=movement['vertical_distance'], vertical_angle=movement['vertical_angle'])
tot_boundary_time += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("%s - was forced by %s and is now at %s" % (self.particle.logstring(), model.__class__.__name__, self.particle.location.logstring()))
self.particle.note = self.particle.outputstring()
# Each timestep, save the particles status and environmental variables.
# This keep fields such as temp, salt, halted, settled, and dead matched up with the number of timesteps
self.particle.save()
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(self.redis_results_channel, json.dumps(self.particle.timestep_dump()))
self.dataset.closenc()
# We won't pull data for the last entry in locations, but we need to populate it with fill data.
self.particle.fill_gap()
if self.usebathy is True:
self._bathymetry.close()
if self.useshore is True:
self._shoreline.close()
logger.info(textwrap.dedent('''Particle %i Stats:
Data read: %f seconds
Model forcing: %s seconds
Boundary intersection: %f seconds''' % (self.particle.uid, tot_read_data, { s: '{:g} seconds'.format(f) for s, f in list(tot_model_time.items()) }, tot_boundary_time)))
return self.particle
def __call__(self, active):
self.active = active
return self.run()
def run(self):
self.load_initial_dataset()
redis_connection = None
if self.redis_url is not None and self.redis_results_channel is not None:
import redis
redis_connection = redis.from_url(self.redis_url)
# Setup shoreline
self._shoreline = None
if self.useshore is True:
self._shoreline = Shoreline(
path=self.shoreline_path,
feature_name=self.shoreline_feature,
point=self.release_location_centroid,
spatialbuffer=self.shoreline_index_buffer)
# Make sure we are not starting on land. Raises exception if we are.
self._shoreline.intersect(
start_point=self.release_location_centroid,
end_point=self.release_location_centroid)
# Setup Bathymetry
if self.usebathy is True:
try:
self._bathymetry = Bathymetry(file=self.bathy_path)
except Exception:
logger.exception(
"Could not load Bathymetry file: %s, using no Bathymetry for this run!"
% self.bathy_path)
self.usebathy = False
# Calculate datetime at every timestep
modelTimestep, newtimes = AsaTransport.get_time_objects_from_model_timesteps(
self.times, start=self.start_time)
if self.time_method == 'interp':
time_indexs = self.timevar.nearest_index(newtimes, select='before')
elif self.time_method == 'nearest':
time_indexs = self.timevar.nearest_index(newtimes)
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
try:
assert len(newtimes) == len(time_indexs)
except AssertionError:
logger.exception(
"Time indexes are messed up. Need to have equal datetime and time indexes"
)
raise
# Keep track of how much time we spend in each area.
tot_boundary_time = 0.
tot_model_time = {}
tot_read_data = 0.
for m in self.models:
tot_model_time[m.name] = 0.
# Set the base conditions
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(self.redis_results_channel,
json.dumps(self.particle.timestep_dump()))
# loop over timesteps
# We don't loop over the last time_index because
# we need to query in the time_index and set the particle's
# location as the 'newtime' object.
for loop_i, i in enumerate(time_indexs[0:-1]):
if self.active and self.active.value is False:
raise ValueError("Particle exiting due to Failure.")
newloc = None
st = time.clock()
# Get the variable data required by the models
if self.time_method == 'nearest':
u, v, w, temp, salt = self.get_nearest_data(i)
elif self.time_method == 'interp':
u, v, w, temp, salt = self.get_linterp_data(
i, newtimes[loop_i])
else:
logger.warn(
"Method for computing u,v,w,temp,salt is unknown. Only 'nearest' and 'interp' are supported."
)
tot_read_data += (time.clock() - st)
# Get the bathy value at the particles location
if self.usebathy is True:
bathymetry_value = self._bathymetry.get_depth(
self.particle.location)
else:
bathymetry_value = -999999999999999
# Age the particle by the modelTimestep (seconds)
# 'Age' meaning the amount of time it has been forced.
self.particle.age(seconds=modelTimestep[loop_i])
# loop over models - sort these in the order you want them to run
for model in self.models:
st = time.clock()
movement = model.move(self.particle,
u,
v,
w,
modelTimestep[loop_i],
temperature=temp,
salinity=salt,
bathymetry_value=bathymetry_value)
newloc = Location4D(latitude=movement['latitude'],
longitude=movement['longitude'],
depth=movement['depth'],
time=newtimes[loop_i + 1])
tot_model_time[m.name] += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - moved %.3f meters (horizontally) and %.3f meters (vertically) by %s with data from %s"
% (self.particle.logstring(), movement['distance'],
movement['vertical_distance'],
model.__class__.__name__,
newtimes[loop_i].isoformat()))
if newloc:
st = time.clock()
self.boundary_interaction(
particle=self.particle,
starting=self.particle.location,
ending=newloc,
distance=movement['distance'],
angle=movement['angle'],
azimuth=movement['azimuth'],
reverse_azimuth=movement['reverse_azimuth'],
vertical_distance=movement['vertical_distance'],
vertical_angle=movement['vertical_angle'])
tot_boundary_time += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - was forced by %s and is now at %s" %
(self.particle.logstring(), model.__class__.__name__,
self.particle.location.logstring()))
self.particle.note = self.particle.outputstring()
# Each timestep, save the particles status and environmental variables.
# This keep fields such as temp, salt, halted, settled, and dead matched up with the number of timesteps
self.particle.save()
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(
self.redis_results_channel,
json.dumps(self.particle.timestep_dump()))
self.dataset.closenc()
# We won't pull data for the last entry in locations, but we need to populate it with fill data.
self.particle.fill_gap()
if self.usebathy is True:
self._bathymetry.close()
if self.useshore is True:
self._shoreline.close()
logger.info(
textwrap.dedent('''Particle %i Stats:
Data read: %f seconds
Model forcing: %s seconds
Boundary intersection: %f seconds''' %
(self.particle.uid, tot_read_data, {
s: '{:g} seconds'.format(f)
for s, f in list(tot_model_time.items())
}, tot_boundary_time)))
return self.particle
def plot_animate(self,
output,
temp_folder=None,
view=(45, -75),
bathy=os.path.join(
__file__,
"../../resources/bathymetry/ETOPO1_Bed_g_gmt4.grd"),
frame_prefix='_paegan',
extent=None,
stride=None,
shore_path=None):
try:
os.mkdirs(os.path.dirname(output))
except:
pass
if not os.path.exists(os.path.dirname(output)):
raise ValueError("Cannot create output directory")
if temp_folder == None:
temp_folder = os.path.dirname(output)
if extent == None:
visual_bbox = (self.nc.variables['lon'][:, 0].min() - .6,
self.nc.variables['lat'][:, 0].min() - .75,
self.nc.variables['lon'][:, 0].max() + .6,
self.nc.variables['lat'][:, 0].max() + .75
) #tracks.buffer(1).bounds
else:
visual_bbox = extent
pt = Point(((visual_bbox[2] - visual_bbox[0]) / 2) + visual_bbox[0],
((visual_bbox[3] - visual_bbox[1]) / 2) + visual_bbox[1])
coast_line = Shoreline(file=shore_path, point=pt,
spatialbuffer=1.5).linestring
c_lons, c_lats = coast_line.xy
c_lons = np.array(c_lons)
c_lats = np.array(c_lats)
c_lons = np.where(
(c_lons >= visual_bbox[0]) & (c_lons <= visual_bbox[2]), c_lons,
np.nan)
c_lats = np.where(
(c_lats >= visual_bbox[1]) & (c_lats <= visual_bbox[3]), c_lats,
np.nan)
#add bathymetry
if stride == None:
if visual_bbox[2] - visual_bbox[0] < 1.5:
stride = 1
else:
stride = 2
nc1 = netCDF4.Dataset(os.path.normpath(bathy))
x = nc1.variables['x']
y = nc1.variables['y']
x_indexes = np.where((x[:] >= visual_bbox[0])
& (x[:] <= visual_bbox[2]))[0]
y_indexes = np.where((y[:] >= visual_bbox[1])
& (y[:] <= visual_bbox[3]))[0]
x_min = x_indexes[0]
x_max = x_indexes[-1]
y_min = y_indexes[0]
y_max = y_indexes[-1]
lons = x[x_min:x_max]
lats = y[y_min:y_max]
bath = nc1.variables['z'][y_min:y_max, x_min:x_max]
bath[bath > 0] = 0
#bath = bath.astype(np.float32)
bath[bath < -800] = -800 #np.nan
x_grid, y_grid = np.meshgrid(lons, lats)
mpl_extent = matplotlib.transforms.Bbox.from_extents(
visual_bbox[0], visual_bbox[1], visual_bbox[2], visual_bbox[3])
CNorm = matplotlib.colors.Normalize(
vmin=-400,
vmax=300,
)
mgr = multiprocessing.Manager()
fname = mgr.list()
p_proj_lons = []
p_proj_lats = []
p_proj_depth = []
lat = self.nc.variables['lat'][:, :]
lon = self.nc.variables['lon'][:, :]
depth = self.nc.variables['depth'][:, :]
time = netCDF4.num2date(self.nc.variables['time'][:],
self.nc.variables['time'].units)
datetimeformat = '%Y-%m-%d %H:%M'
p = []
c = 0
length = self.nc.variables['particle'].shape[0]
def render_frame(visual_bbox, c_lons, c_lat, mpl_extent, x_grid,
y_grid, bath, stride, view, length, lat, lon, depth,
temp_folder, frame_prefix, c, semo):
import numpy as np
import netCDF4, sys, os
import matplotlib
import matplotlib.pyplot
from matplotlib import cm, animation
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.ticker import MultipleLocator
with semo:
fig2 = matplotlib.pyplot.figure(figsize=(12, 6))
ax2 = fig2.add_subplot(111, projection='3d')
ax3 = fig2.add_axes([.75, .1, .15, .3])
ax4 = fig2.add_axes([.2, .1, .15, .3])
subbox = visual_bbox #(self.nc.variables['lon'][:,0].min(), self.nc.variables['lat'][:,0].min(),
#self.nc.variables['lon'][:,0].max(), self.nc.variables['lat'][:,0].max())
ax3.plot(c_lons,
c_lats,
clip_box=mpl_extent,
clip_on=True,
color='c') # shoreline
ax3.set_xlim(subbox[0], subbox[2])
ax3.set_ylim(subbox[1], subbox[3])
#ax3.pcolor(x_grid, y_grid, bath, cmap="Blues_r", norm=CNorm)
ax2.plot_surface(x_grid,
y_grid,
bath,
rstride=stride,
cstride=stride,
cmap="Blues_r",
linewidth=0.01,
antialiased=False,
norm=CNorm,
shade=True,
edgecolor='#6183A6')
ax2.plot(c_lons, c_lats, np.zeros_like(c_lons))
ax2.set_xlim3d(visual_bbox[0], visual_bbox[2])
ax2.set_ylim3d(visual_bbox[1], visual_bbox[3])
ax2.view_init(*view)
ax2.set_title(time[i].strftime(datetimeformat) + " - " +
time[i + 2].strftime(datetimeformat))
#ax2.set_zmargin(50)
ax3.set_xlabel('Longitude')
ax3.set_ylabel('Latitude')
ax3.tick_params(axis='both', which='major', labelsize=10)
ax3.yaxis.set_ticks_position('right')
ax3.ticklabel_format(axis='x', style='plain')
ax3.xaxis.set_major_locator(MultipleLocator(.5))
ax3.grid(True)
ax4.set_ylabel('Depth (m)')
ax4.set_ylim(-200, 0)
ax4.tick_params(axis='x', which='major', labelsize=10)
#ax4.set_xlim(1,3)
ax4.xaxis.set_ticklabels([])
#ax3.xaxis.set_ticklabels(np.unique(c_lons.astype(int)))
ax2.set_zlabel('Depth (m)')
#ax2.set_frame_on(False)
#ax2.set_position([0,0,1,1])
ax2.xaxis.set_ticklabels([])
ax2.yaxis.set_ticklabels([])
#ax2.zaxis.set_ticklabels(['Surface'])
ax2.zaxis.set_ticks(range(-800, 100, 200))
ax2.grid(False)
#ax2.set_zlim(-200, 100)
for j in range(length):
# Each particle
ax2.plot(lon[i:i + 3, j],
lat[i:i + 3, j],
depth[i:i + 3, j],
':',
c='r',
linewidth=2,
markersize=5,
markerfacecolor='r')
ax3.plot(
lon[:i + 3, j],
lat[:i + 3, j],
c='.2',
linewidth=.5,
markersize=5,
markerfacecolor='r',
)
ax3.scatter(lon[i + 2, j], lat[i + 2, j], c='r')
ax4.plot(range(i + 3),
depth[:i + 3, j],
c='r',
linewidth=.5,
aa=True)
ax4.scatter(np.ones_like(depth[i + 2, j]) * (i + 2),
depth[i + 2, j],
c='r')
if i == 2:
ax4.set_xlim(i - 2, i + 2.25)
elif i >= 3:
ax4.set_xlim(i - 3, i + 2.25)
else:
ax4.set_xlim(i, i + 2.25)
#ax2.scatter(lon[i,:], lat[i,:], depth[i,:], zdir='z', c='r')
ax2.set_zlim3d(-800, 25)
image_path = os.path.join(temp_folder,
'%s%04d.png' % (frame_prefix, c))
fig2.savefig(image_path, dpi=350, bbox_inches='tight')
fname.append(image_path)
del ax2, ax3, ax4, subbox, fig2
jobs = []
for i in range(self.nc.variables['time'].shape[0])[:-4:2]:
p = multiprocessing.Process(
target=render_frame,
args=(visual_bbox, c_lons, c_lats, mpl_extent, x_grid, y_grid,
bath, stride, view, length, lat, lon, depth, temp_folder,
frame_prefix, c, semo))
p.start()
jobs.append(p)
c += 1
for j in jobs:
j.join(120)
return save_animation(output, sorted(fname), frame_prefix=frame_prefix)
def __call__(self, proc, active):
self.active = active
if self.usebathy == True:
self._bathymetry = Bathymetry(file=self.bathy)
self._shoreline = None
if self.useshore == True:
self._shoreline = Shoreline(file=self.shoreline_path, point=self.release_location_centroid, spatialbuffer=0.25)
# Make sure we are not starting on land. Raises exception if we are.
self._shoreline.intersect(start_point=self.release_location_centroid, end_point=self.release_location_centroid)
self.proc = proc
part = self.part
if self.active.value == True:
while self.get_data.value == True:
logger.debug("Waiting for DataController to start...")
timer.sleep(10)
pass
# Initialize commondataset of local cache, then
# close the related netcdf file
try:
with self.read_lock:
self.read_count.value += 1
self.has_read_lock.append(os.getpid())
self.dataset = CommonDataset.open(self.localpath)
self.dataset.closenc()
except StandardError:
logger.warn("No cache file: %s. Particle exiting" % self.localpath)
raise
finally:
with self.read_lock:
self.read_count.value -= 1
self.has_read_lock.remove(os.getpid())
# Calculate datetime at every timestep
modelTimestep, newtimes = AsaTransport.get_time_objects_from_model_timesteps(self.times, start=self.start_time)
# Load Timevar from pickle serialization
f = open(self.timevar_pickle_path,"rb")
timevar = pickle.load(f)
f.close()
if self.time_method == 'interp':
time_indexs = timevar.nearest_index(newtimes, select='before')
elif self.time_method == 'nearest':
time_indexs = timevar.nearest_index(newtimes)
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
try:
assert len(newtimes) == len(time_indexs)
except AssertionError:
logger.error("Time indexes are messed up. Need to have equal datetime and time indexes")
raise
# loop over timesteps
# We don't loop over the last time_index because
# we need to query in the time_index and set the particle's
# location as the 'newtime' object.
for loop_i, i in enumerate(time_indexs[0:-1]):
if self.active.value == False:
raise ValueError("Particle exiting due to Failure.")
newloc = None
# if need a time that is outside of what we have
#if self.active.value == True:
# while self.get_data.value == True:
# logger.info("Waiting for DataController to get out...")
# timer.sleep(4)
# pass
# Get the variable data required by the models
if self.time_method == 'nearest':
u, v, w, temp, salt = self.data_nearest(i, newtimes[loop_i])
elif self.time_method == 'interp':
u, v, w, temp, salt = self.data_interp(i, timevar, newtimes[loop_i])
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
#logger.info("U: %.4f, V: %.4f, W: %.4f" % (u,v,w))
#logger.info("Temp: %.4f, Salt: %.4f" % (temp,salt))
# Get the bathy value at the particles location
if self.usebathy == True:
bathymetry_value = self._bathymetry.get_depth(part.location)
else:
bathymetry_value = -999999999999999
# Age the particle by the modelTimestep (seconds)
# 'Age' meaning the amount of time it has been forced.
part.age(seconds=modelTimestep[loop_i])
# loop over models - sort these in the order you want them to run
for model in self.models:
movement = model.move(part, u, v, w, modelTimestep[loop_i], temperature=temp, salinity=salt, bathymetry_value=bathymetry_value)
newloc = Location4D(latitude=movement['latitude'], longitude=movement['longitude'], depth=movement['depth'], time=newtimes[loop_i+1])
logger.debug("%s - moved %.3f meters (horizontally) and %.3f meters (vertically) by %s with data from %s" % (part.logstring(), movement['distance'], movement['vertical_distance'], model.__class__.__name__, newtimes[loop_i].isoformat()))
if newloc:
self.boundary_interaction(particle=part, starting=part.location, ending=newloc,
distance=movement['distance'], angle=movement['angle'],
azimuth=movement['azimuth'], reverse_azimuth=movement['reverse_azimuth'],
vertical_distance=movement['vertical_distance'], vertical_angle=movement['vertical_angle'])
logger.debug("%s - was forced by %s and is now at %s" % (part.logstring(), model.__class__.__name__, part.location.logstring()))
part.note = part.outputstring()
# Each timestep, save the particles status and environmental variables.
# This keep fields such as temp, salt, halted, settled, and dead matched up with the number of timesteps
part.save()
# We won't pull data for the last entry in locations, but we need to populate it with fill data.
part.fill_environment_gap()
if self.usebathy == True:
self._bathymetry.close()
if self.useshore == True:
self._shoreline.close()
return part
def test_get_feature_type_info(self):
s = Shoreline()
assert isinstance(dict, s.get_capabilities())
def test_get_capabilities(self):
s = Shoreline()
assert isinstance(dict, s.get_capabilities())
class BaseForcer(object):
def __init__(self, hydrodataset, **kwargs):
"""
part, common_variables, timevar, times, start_time, models,
release_location_centroid, usebathy, useshore, usesurface,
get_data, n_run, read_lock, has_read_lock, read_count,
point_get, data_request_lock, has_data_request_lock, reverse_distance=None, bathy=None,
shoreline_path=None, shoreline_feature=None, time_method=None, caching=None, redis_url=None, redis_results_channel=None, shoreline_index_buffer=None):
This is the task/class/object/job that forces an
individual particle and communicates with the
other particles and data controller for local
cache updates
"""
assert hydrodataset is not None
# Common parameters
self.hydrodataset = hydrodataset
self.bathy_path = kwargs.get("bathy_path")
self.release_location_centroid = kwargs.get(
"release_location_centroid")
self.particle = kwargs.get("particle")
self.times = kwargs.get("times")
self.timevar = kwargs.get("timevar", None)
self.start_time = kwargs.get("start_time")
self.models = kwargs.get("models", [])
self.usebathy = kwargs.get("usebathy", False)
self.useshore = kwargs.get("useshore", False)
self.usesurface = kwargs.get("usesurface", True)
self.shoreline_path = kwargs.get("shoreline_path")
self.shoreline_feature = kwargs.get("shoreline_feature", None)
self.shoreline_index_buffer = kwargs.get("shoreline_index_buffer", 0.1)
self.time_method = kwargs.get("time_method", "nearest")
self.reverse_distance = kwargs.get("reverse_distance", 500)
# Redis for results
self.redis_url = kwargs.get("redis_url", None)
self.redis_results_channel = kwargs.get("redis_results_channel", None)
# Set common variable names
self.common_variables = kwargs.get("common_variables")
self.uname = self.common_variables.get("u", None)
self.vname = self.common_variables.get("v", None)
self.wname = self.common_variables.get("w", None)
self.temp_name = self.common_variables.get("temp", None)
self.salt_name = self.common_variables.get("salt", None)
self.xname = self.common_variables.get("x", None)
self.yname = self.common_variables.get("y", None)
self.zname = self.common_variables.get("z", None)
self.tname = self.common_variables.get("time", None)
self.active = None
def load_initial_dataset(self):
"""
Initialize self.dataset, then close it
A cacher will have to wrap this in locks, while a straight runner will not.
"""
try:
self.dataset = CommonDataset.open(self.hydrodataset)
if self.timevar is None:
self.timevar = self.dataset.gettimevar(
self.common_variables.get("u"))
except Exception:
logger.warn("No source dataset: %s. Particle exiting" %
self.hydrodataset)
raise
def boundary_interaction(self, **kwargs):
"""
Returns a list of Location4D objects
"""
particle = kwargs.pop('particle')
starting = kwargs.pop('starting')
ending = kwargs.pop('ending')
# shoreline
if self.useshore:
intersection_point = self._shoreline.intersect(
start_point=starting.point, end_point=ending.point)
if intersection_point is not None:
# Set the intersection point.
hitpoint = Location4D(point=intersection_point['point'],
time=starting.time +
(ending.time - starting.time))
particle.location = hitpoint
# This relies on the shoreline to put the particle in water and not on shore.
resulting_point = self._shoreline.react(
start_point=starting,
end_point=ending,
hit_point=hitpoint,
reverse_distance=self.reverse_distance,
feature=intersection_point['feature'],
distance=kwargs.get('distance'),
angle=kwargs.get('angle'),
azimuth=kwargs.get('azimuth'),
reverse_azimuth=kwargs.get('reverse_azimuth'))
ending.latitude = resulting_point.latitude
ending.longitude = resulting_point.longitude
ending.depth = resulting_point.depth
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - hit the shoreline at %s. Setting location to %s."
% (particle.logstring(), hitpoint.logstring(),
ending.logstring()))
# bathymetry
if self.usebathy:
if not particle.settled:
bintersect = self._bathymetry.intersect(start_point=starting,
end_point=ending)
if bintersect:
pt = self._bathymetry.react(type='reverse',
start_point=starting,
end_point=ending)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - hit the bottom at %s. Setting location to %s."
% (particle.logstring(), ending.logstring(),
pt.logstring()))
ending.latitude = pt.latitude
ending.longitude = pt.longitude
ending.depth = pt.depth
# sea-surface
if self.usesurface:
if ending.depth > 0:
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - rose out of the water. Setting depth to 0." %
particle.logstring())
ending.depth = 0
particle.location = ending
def get_nearest_data(self, i):
""" Note: self.dataset.opennc() must be called before calling this function.
This is because the caching forcer must close it everytime, while a non caching
forcer can leave the dataset open.
"""
try:
# Grab data at time index closest to particle location
u = np.mean(
np.mean(
self.dataset.get_values('u',
timeinds=[np.asarray([i])],
point=self.particle.location)))
v = np.mean(
np.mean(
self.dataset.get_values('v',
timeinds=[np.asarray([i])],
point=self.particle.location)))
# if there is vertical velocity inthe dataset, get it
if 'w' in self.dataset.nc.variables:
w = np.mean(
np.mean(
self.dataset.get_values('w',
timeindsf=[np.asarray([i])],
point=self.particle.location)))
else:
w = 0.0
# If there is salt and temp in the dataset, get it
if self.temp_name is not None and self.salt_name is not None:
temp = np.mean(
np.mean(
self.dataset.get_values('temp',
timeinds=[np.asarray([i])],
point=self.particle.location)))
salt = np.mean(
np.mean(
self.dataset.get_values('salt',
timeinds=[np.asarray([i])],
point=self.particle.location)))
# Check for nans that occur in the ocean (happens because
# of model and coastline resolution mismatches)
if np.isnan(u).any() or np.isnan(v).any() or np.isnan(w).any():
# Take the mean of the closest 4 points
# If this includes nan which it will, result is nan
uarray1 = self.dataset.get_values('u',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
varray1 = self.dataset.get_values('v',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
if 'w' in self.dataset.nc.variables:
warray1 = self.dataset.get_values(
'w',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
w = warray1.mean()
else:
w = 0.0
if self.temp_name is not None and self.salt_name is not None:
temparray1 = self.dataset.get_values(
'temp',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
saltarray1 = self.dataset.get_values(
'salt',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
temp = temparray1.mean()
salt = saltarray1.mean()
u = uarray1.mean()
v = varray1.mean()
if self.temp_name is None:
temp = np.nan
if self.salt_name is None:
salt = np.nan
except Exception:
logger.exception("Could not retrieve data.")
raise
return u, v, w, temp, salt
def get_linterp_data(self, i, currenttime):
""" Note: self.dataset.opennc() must be called before calling this function.
This is because the caching forcer must close it everytime, while a non caching
forcer can leave the dataset open.
"""
try:
# Grab data at time index closest to particle location
u = [
np.mean(
np.mean(
self.dataset.get_values(
'u',
timeinds=[np.asarray([i])],
point=self.particle.location))),
np.mean(
np.mean(
self.dataset.get_values('u',
timeinds=[np.asarray([i + 1])],
point=self.particle.location)))
]
v = [
np.mean(
np.mean(
self.dataset.get_values(
'v',
timeinds=[np.asarray([i])],
point=self.particle.location))),
np.mean(
np.mean(
self.dataset.get_values('v',
timeinds=[np.asarray([i + 1])],
point=self.particle.location)))
]
# if there is vertical velocity inthe dataset, get it
if 'w' in self.dataset.nc.variables:
w = [
np.mean(
np.mean(
self.dataset.get_values(
'w',
timeinds=[np.asarray([i])],
point=self.particle.location))),
np.mean(
np.mean(
self.dataset.get_values(
'w',
timeinds=[np.asarray([i + 1])],
point=self.particle.location)))
]
else:
w = [0.0, 0.0]
# If there is salt and temp in the dataset, get it
if self.temp_name is not None and self.salt_name is not None:
temp = [
np.mean(
np.mean(
self.dataset.get_values(
'temp',
timeinds=[np.asarray([i])],
point=self.particle.location))),
np.mean(
np.mean(
self.dataset.get_values(
'temp',
timeinds=[np.asarray([i + 1])],
point=self.particle.location)))
]
salt = [
np.mean(
np.mean(
self.dataset.get_values(
'salt',
timeinds=[np.asarray([i])],
point=self.particle.location))),
np.mean(
np.mean(
self.dataset.get_values(
'salt',
timeinds=[np.asarray([i + 1])],
point=self.particle.location)))
]
# Check for nans that occur in the ocean (happens because
# of model and coastline resolution mismatches)
if np.isnan(u).any() or np.isnan(v).any() or np.isnan(w).any():
# Take the mean of the closest 4 points
# If this includes nan which it will, result is nan
uarray1 = self.dataset.get_values('u',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
varray1 = self.dataset.get_values('v',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
uarray2 = self.dataset.get_values(
'u',
timeinds=[np.asarray([i + 1])],
point=self.particle.location,
num=2)
varray2 = self.dataset.get_values(
'v',
timeinds=[np.asarray([i + 1])],
point=self.particle.location,
num=2)
if 'w' in self.dataset.nc.variables:
warray1 = self.dataset.get_values(
'w',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
warray2 = self.dataset.get_values(
'w',
timeinds=[np.asarray([i + 1])],
point=self.particle.location,
num=2)
w = [warray1.mean(), warray2.mean()]
else:
w = [0.0, 0.0]
if self.temp_name is not None and self.salt_name is not None:
temparray1 = self.dataset.get_values(
'temp',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
saltarray1 = self.dataset.get_values(
'salt',
timeinds=[np.asarray([i])],
point=self.particle.location,
num=2)
temparray2 = self.dataset.get_values(
'temp',
timeinds=[np.asarray([i + 1])],
point=self.particle.location,
num=2)
saltarray2 = self.dataset.get_values(
'salt',
timeinds=[np.asarray([i + 1])],
point=self.particle.location,
num=2)
temp = [temparray1.mean(), temparray2.mean()]
salt = [saltarray1.mean(), saltarray2.mean()]
u = [uarray1.mean(), uarray2.mean()]
v = [varray1.mean(), varray2.mean()]
# Linear interp of data between timesteps
currenttime = date2num(currenttime)
timevar = self.timevar.datenum
u = self.linterp(timevar[i:i + 2], u, currenttime)
v = self.linterp(timevar[i:i + 2], v, currenttime)
w = self.linterp(timevar[i:i + 2], w, currenttime)
if self.temp_name is not None and self.salt_name is not None:
temp = self.linterp(timevar[i:i + 2], temp, currenttime)
salt = self.linterp(timevar[i:i + 2], salt, currenttime)
if self.temp_name is None:
temp = np.nan
if self.salt_name is None:
salt = np.nan
except Exception:
logger.exception("Could not retrieve data.")
raise
return u, v, w, temp, salt
def linterp(self, setx, sety, x):
"""
Linear interp of model data values between time steps
"""
if math.isnan(sety[0]) or math.isnan(setx[0]):
return np.nan
return sety[0] + (x - setx[0]) * ((sety[1] - sety[0]) /
(setx[1] - setx[0]))
def run(self):
self.load_initial_dataset()
redis_connection = None
if self.redis_url is not None and self.redis_results_channel is not None:
import redis
redis_connection = redis.from_url(self.redis_url)
# Setup shoreline
self._shoreline = None
if self.useshore is True:
self._shoreline = Shoreline(
path=self.shoreline_path,
feature_name=self.shoreline_feature,
point=self.release_location_centroid,
spatialbuffer=self.shoreline_index_buffer)
# Make sure we are not starting on land. Raises exception if we are.
self._shoreline.intersect(
start_point=self.release_location_centroid,
end_point=self.release_location_centroid)
# Setup Bathymetry
if self.usebathy is True:
try:
self._bathymetry = Bathymetry(file=self.bathy_path)
except Exception:
logger.exception(
"Could not load Bathymetry file: %s, using no Bathymetry for this run!"
% self.bathy_path)
self.usebathy = False
# Calculate datetime at every timestep
modelTimestep, newtimes = AsaTransport.get_time_objects_from_model_timesteps(
self.times, start=self.start_time)
if self.time_method == 'interp':
time_indexs = self.timevar.nearest_index(newtimes, select='before')
elif self.time_method == 'nearest':
time_indexs = self.timevar.nearest_index(newtimes)
else:
logger.warn("Method for computing u,v,w,temp,salt not supported!")
try:
assert len(newtimes) == len(time_indexs)
except AssertionError:
logger.exception(
"Time indexes are messed up. Need to have equal datetime and time indexes"
)
raise
# Keep track of how much time we spend in each area.
tot_boundary_time = 0.
tot_model_time = {}
tot_read_data = 0.
for m in self.models:
tot_model_time[m.name] = 0.
# Set the base conditions
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(self.redis_results_channel,
json.dumps(self.particle.timestep_dump()))
# loop over timesteps
# We don't loop over the last time_index because
# we need to query in the time_index and set the particle's
# location as the 'newtime' object.
for loop_i, i in enumerate(time_indexs[0:-1]):
if self.active and self.active.value is False:
raise ValueError("Particle exiting due to Failure.")
newloc = None
st = time.clock()
# Get the variable data required by the models
if self.time_method == 'nearest':
u, v, w, temp, salt = self.get_nearest_data(i)
elif self.time_method == 'interp':
u, v, w, temp, salt = self.get_linterp_data(
i, newtimes[loop_i])
else:
logger.warn(
"Method for computing u,v,w,temp,salt is unknown. Only 'nearest' and 'interp' are supported."
)
tot_read_data += (time.clock() - st)
# Get the bathy value at the particles location
if self.usebathy is True:
bathymetry_value = self._bathymetry.get_depth(
self.particle.location)
else:
bathymetry_value = -999999999999999
# Age the particle by the modelTimestep (seconds)
# 'Age' meaning the amount of time it has been forced.
self.particle.age(seconds=modelTimestep[loop_i])
# loop over models - sort these in the order you want them to run
for model in self.models:
st = time.clock()
movement = model.move(self.particle,
u,
v,
w,
modelTimestep[loop_i],
temperature=temp,
salinity=salt,
bathymetry_value=bathymetry_value)
newloc = Location4D(latitude=movement['latitude'],
longitude=movement['longitude'],
depth=movement['depth'],
time=newtimes[loop_i + 1])
tot_model_time[m.name] += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - moved %.3f meters (horizontally) and %.3f meters (vertically) by %s with data from %s"
% (self.particle.logstring(), movement['distance'],
movement['vertical_distance'],
model.__class__.__name__,
newtimes[loop_i].isoformat()))
if newloc:
st = time.clock()
self.boundary_interaction(
particle=self.particle,
starting=self.particle.location,
ending=newloc,
distance=movement['distance'],
angle=movement['angle'],
azimuth=movement['azimuth'],
reverse_azimuth=movement['reverse_azimuth'],
vertical_distance=movement['vertical_distance'],
vertical_angle=movement['vertical_angle'])
tot_boundary_time += (time.clock() - st)
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
"%s - was forced by %s and is now at %s" %
(self.particle.logstring(), model.__class__.__name__,
self.particle.location.logstring()))
self.particle.note = self.particle.outputstring()
# Each timestep, save the particles status and environmental variables.
# This keep fields such as temp, salt, halted, settled, and dead matched up with the number of timesteps
self.particle.save()
# If using Redis, send the results
if redis_connection is not None:
redis_connection.publish(
self.redis_results_channel,
json.dumps(self.particle.timestep_dump()))
self.dataset.closenc()
# We won't pull data for the last entry in locations, but we need to populate it with fill data.
self.particle.fill_gap()
if self.usebathy is True:
self._bathymetry.close()
if self.useshore is True:
self._shoreline.close()
logger.info(
textwrap.dedent('''Particle %i Stats:
Data read: %f seconds
Model forcing: %s seconds
Boundary intersection: %f seconds''' %
(self.particle.uid, tot_read_data, {
s: '{:g} seconds'.format(f)
for s, f in list(tot_model_time.items())
}, tot_boundary_time)))
return self.particle
def __call__(self, active):
self.active = active
return self.run()
