def setUp(self): start_lat = 38 start_lon = -76 start_depth = -5 temp_time = datetime.utcnow() self.start_time = datetime(temp_time.year, temp_time.month, temp_time.day, temp_time.hour) self.loc = Location4D(latitude=start_lat, longitude=start_lon, depth=start_depth, time=self.start_time) # Generate time,u,v,z as randoms # 48 timesteps at an hour each = 2 days of running self.times = list(range(0, 172800, 3600)) # in seconds self.u = [] self.v = [] self.z = [] for w in range(0, 48): self.z.append(random.gauss(0, 0.0001)) # gaussian in m/s self.u.append(abs(AsaRandom.random())) # random function in m/s self.v.append(abs(AsaRandom.random())) # random function in m/s self.particles = [] # Create particles for i in range(0, 3): p = Particle() p.location = self.loc self.particles.append(p) # Create a transport instance with horiz and vert dispersions self.transport_model = Transport(horizDisp=0.05, vertDisp=0.00003)
def setUp(self): start_lat = 38 start_lon = -76 start_depth = -5 temp_time = datetime.utcnow() self.start_time = datetime(temp_time.year, temp_time.month, temp_time.day, temp_time.hour) self.loc = Location4D(latitude=start_lat, longitude=start_lon, depth=start_depth, time=self.start_time) # Generate time,u,v,z as randoms # 48 timesteps at an hour each = 2 days of running self.times = range(0, 172800, 3600) # in seconds self.u = [] self.v = [] self.z = [] for w in xrange(0, 48): self.z.append(random.gauss(0, 0.0001)) # gaussian in m/s self.u.append(abs(AsaRandom.random())) # random function in m/s self.v.append(abs(AsaRandom.random())) # random function in m/s self.particles = [] # Create particles for i in xrange(0, 3): p = Particle() p.location = self.loc self.particles.append(p) # Create a transport instance with horiz and vert dispersions self.transport_model = Transport(horizDisp=0.05, vertDisp=0.00003)
def move(self, particle, u, v, w, modelTimestep, **kwargs): """ Returns the lat, lon, H, and velocity of a projected point given a starting lat and lon (dec deg), a depth (m) below sea surface (positive up), u, v, and w velocity components (m/s), a horizontal and vertical displacement coefficient (m^2/s) H (m), and a model timestep (s). GreatCircle calculations are done based on the Vincenty Direct method. Returns a dict like: { 'latitude': x, 'azimuth': x, 'reverse_azimuth': x, 'longitude': x, 'depth': x, 'u': x 'v': x, 'w': x, 'distance': x, 'angle': x, 'vertical_distance': x, 'vertical_angle': x } """ logger.debug("U: %s, V: %s, W: %s" % (str(u), str(v), str(w))) # IMPORTANT: # If we got no data from the model, we are using the last available value stored in the particles! if (u is None) or (u is not None and math.isnan(u)): u = particle.last_u() if (v is None) or (v is not None and math.isnan(v)): v = particle.last_v() if (w is None) or (w is not None and math.isnan(w)): w = particle.last_w() particle.u_vector = u particle.v_vector = v particle.w_vector = w if particle.halted: u, v, w = 0, 0, 0 else: u += AsaRandom.random() * ((2 * self._horizDisp / modelTimestep)** 0.5) # u transformation calcualtions v += AsaRandom.random() * ((2 * self._horizDisp / modelTimestep)** 0.5) # v transformation calcualtions w += AsaRandom.random() * ((2 * self._vertDisp / modelTimestep)** 0.5) # w transformation calculations result = AsaTransport.distance_from_location_using_u_v_w( u=u, v=v, w=w, timestep=modelTimestep, location=particle.location) result['u'] = u result['v'] = v result['w'] = w return result
def move(self, particle, u, v, w, modelTimestep, **kwargs): """ Returns the lat, lon, H, and velocity of a projected point given a starting lat and lon (dec deg), a depth (m) below sea surface (positive up), u, v, and w velocity components (m/s), a horizontal and vertical displacement coefficient (m^2/s) H (m), and a model timestep (s). GreatCircle calculations are done based on the Vincenty Direct method. Returns a dict like: { 'latitude': x, 'azimuth': x, 'reverse_azimuth': x, 'longitude': x, 'depth': x, 'u': x 'v': x, 'w': x, 'distance': x, 'angle': x, 'vertical_distance': x, 'vertical_angle': x } """ logger.debug("U: %s, V: %s, W: %s" % (str(u),str(v),str(w))) # IMPORTANT: # If we got no data from the model, we are using the last available value stored in the particles! if (u is None) or (u is not None and math.isnan(u)): u = particle.last_u() if (v is None) or (v is not None and math.isnan(v)): v = particle.last_v() if (w is None) or (w is not None and math.isnan(w)): w = particle.last_w() particle.u_vector = u particle.v_vector = v particle.w_vector = w if particle.halted: u,v,w = 0,0,0 else: u += AsaRandom.random() * ((2 * self._horizDisp / modelTimestep) ** 0.5) # u transformation calcualtions v += AsaRandom.random() * ((2 * self._horizDisp / modelTimestep) ** 0.5) # v transformation calcualtions w += AsaRandom.random() * ((2 * self._vertDisp / modelTimestep) ** 0.5) # w transformation calculations result = AsaTransport.distance_from_location_using_u_v_w(u=u, v=v, w=w, timestep=modelTimestep, location=particle.location) result['u'] = u result['v'] = v result['w'] = w return result
def __reverse(self, **kwargs): """ Reverse particle just off of the shore in the direction that it came in. Adds a slight random factor to the distance and angle it is reversed in. """ #st = time.clock() start_point = kwargs.pop('start_point') hit_point = kwargs.pop('hit_point') reverse_azimuth = kwargs.pop('reverse_azimuth') reverse_distance = kwargs.get('reverse_distance', None) if reverse_distance is None: reverse_distance = 100 # Randomize the reverse angle slightly (+/- 5 degrees) random_azimuth = reverse_azimuth + AsaRandom.random() * 5 count = 0 nudge_distance = 0.01 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) # Find point just offshore to do testing with. Try 15 times (~350m). This makes sure the start_point is in the water # for the next call to intersect (next while loop). while self.intersect(single_point=nudge_loc.point) and count < 16: nudge_distance *= 2 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) count += 1 # We tried 16 times and couldn't find a point. This should totally never happen. if count == 16: logger.warn("LOOK: Could not find location in water to do shoreline calculation with. Assuming particle did not move from original location") return start_point # Keep trying to throw particle back, halfing the distance each time until it is in water. # Only half it 6 times before giving up and returning the point which the particle came from. count = 0 # Distance amount to half each iteration changing_distance = reverse_distance new_point = AsaGreatCircle.great_circle(distance=reverse_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) # We don't want to reverse further than the current spatial buffer, because we will reindex the # source file everytime we reverse, which will slow down the calculations considerably. while (not self._spatial_query_object.contains(new_loc.point) or self.intersect(start_point=nudge_loc.point, end_point=new_loc.point)) and count < 6: changing_distance /= 2 new_point = AsaGreatCircle.great_circle(distance=changing_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) count += 1 # We tried 10 times and the particle was still on shore, return the point the particle started from. # No randomization. if count == 6: logger.warn("LOOK: Could not react particle with shoreline. Assuming particle did not move from original location") return start_point #logger.info("Reaction time: %f" % (time.clock() - st)) return new_loc
def test_create_random_filename(self): temp_filename = AsaRandom.filename(prefix="superduper", suffix=".nc") path = urlparse(temp_filename).path name, ext = os.path.splitext(path) assert name.index("superduper") == 0 assert ext == ".nc"
def __reverse(self, **kwargs): """ Reverse particle just off of the shore in the direction that it came in. Adds a slight random factor to the distance and angle it is reversed in. """ start_point = kwargs.pop('start_point') hit_point = kwargs.pop('hit_point') distance = kwargs.pop('distance') azimuth = kwargs.pop('azimuth') reverse_azimuth = kwargs.pop('reverse_azimuth') reverse_distance = kwargs.get('reverse_distance', None) if reverse_distance is None: reverse_distance = 100 # Randomize the reverse angle slightly (+/- 5 degrees) random_azimuth = reverse_azimuth + AsaRandom.random() * 5 count = 0 nudge_distance = 0.01 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) # Find point just offshore to do testing with. Try 15 times (~350m). This makes sure the start_point is in the water # for the next call to intersect (next while loop). while self.intersect(single_point=nudge_loc.point) and count < 16: nudge_distance *= 2 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) count += 1 # We tried 16 times and couldn't find a point. This should totally never happen. if count == 16: logger.debug("WOW. Could not find location in water to do shoreline calculation with. Assuming particle did not move from original location") return start_point # Keep trying to throw particle back, halfing the distance each time until it is in water. # Only half it 12 times before giving up and returning the point which the particle came from. count = 0 # Distance amount to half each iteration changing_distance = reverse_distance new_point = AsaGreatCircle.great_circle(distance=reverse_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) while self.intersect(start_point=nudge_loc.point, end_point=new_loc.point) and count < 12: changing_distance /= 2 new_point = AsaGreatCircle.great_circle(distance=changing_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) count += 1 # We tried 10 times and the particle was still on shore, return the point the particle started from. # No randomization. if count == 12: logger.debug("Could not react particle with shoreline. Assuming particle did not move from original location") return start_point return new_loc
def setup_run(self, **kwargs): super(CachingModelController, self).setup_run(**kwargs) # Should we remove the cache file at the end of the run? self.remove_cache = kwargs.get("remove_cache", False) self.cache_path = kwargs.get("cache_path", None) # Create a temp file for the cache if nothing was passed in if self.cache_path is None: default_cache_dir = os.path.join(os.path.dirname(__file__), "_cache") temp_name = AsaRandom.filename(prefix=str(datetime.now().microsecond), suffix=".nc") self.cache_path = os.path.join(default_cache_dir, temp_name) # Be sure the cache directory exists if not os.path.exists(os.path.dirname(self.cache_path)): logger.info("Creating cache directory: %s" % self.cache_path) os.makedirs(os.path.dirname(self.cache_path)) # Create the shared state objects # Particles use this to tell the Data Controller to "get_data". # The CachingDataController sets this to False when it is done writing to the cache file. # Particles will wait for this to be False before reading from the cache file. # If we are caching, this starts as True so the Particles don't take off. If we # are not caching, this is False so the Particles can start immediatly. self.get_data = self.mgr.Value('bool', True) # Particles use this to tell the DataContoller which indices to 'get_data' for self.point_get = self.mgr.Value('list', [0, 0, 0]) # This locks access to the 'has_data_request_lock' value self.data_request_lock = self.mgr.Lock() # This tracks which Particle PID is asking the CachingDataController for data self.has_data_request_lock = self.mgr.Value('int', -1) # The lock that controls access to modifying 'has_read_lock' and 'read_count' self.read_lock = self.mgr.Lock() # List of Particle PIDs that are reading from the cache self.has_read_lock = self.mgr.list() # The number of Particles that are reading from the cache self.read_count = self.mgr.Value('int', 0) # When something is writing to the cache file self.write_lock = self.mgr.Lock() # PID of process with lock self.has_write_lock = self.mgr.Value('int', -1)
def __reverse(self, **kwargs): """ Reverse particle just off of the shore in the direction that it came in. Adds a slight random factor to the distance and angle it is reversed in. """ start_point = kwargs.pop('start_point') hit_point = kwargs.pop('hit_point') distance = kwargs.pop('distance') azimuth = kwargs.pop('azimuth') reverse_azimuth = kwargs.pop('reverse_azimuth') reverse_distance = kwargs.get('reverse_distance', None) if reverse_distance is None: reverse_distance = 100 # Randomize the reverse angle slightly (+/- 5 degrees) random_azimuth = reverse_azimuth + AsaRandom.random() * 5 # Nudge the hitpoint off of the shore by a tiny bit to test shoreline intersection in while loop. nudged_hit_point = AsaGreatCircle.great_circle(distance=0.01, azimuth=random_azimuth, start_point=hit_point) nudged_hit_location = Location4D(latitude=nudged_hit_point['latitude'], longitude=nudged_hit_point['longitude'], depth=start_point.depth) new_point = AsaGreatCircle.great_circle(distance=reverse_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) # Keep trying to throw particle back, halfing the distance each time until it is in water. # Only half it 10 times before giving up and returning the point which the particle came from. count = 0 # Distance amount to half each iteration changing_distance = reverse_distance while self.intersect(start_point=nudged_hit_location.point, end_point=new_loc.point) and count < 10: changing_distance /= 2 new_point = AsaGreatCircle.great_circle(distance=changing_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) count += 1 # We tried 10 times and the particle was still on shore, return the point the particle started from. # No randomization. if count == 10: logger.warn("Could not react particle with shoreline. Assuming particle did not move from original location") new_loc = start_point return new_loc
def run(self, hydrodataset, **kwargs): # Add ModelController description to logfile logger.info(self) # Add the model descriptions to logfile for m in self._models: logger.info(m) # Calculate the model timesteps # We need times = len(self._nstep) + 1 since data is stored one timestep # after a particle is forced with the final timestep's data. times = range(0, (self._step * self._nstep) + 1, self._step) # Calculate a datetime object for each model timestep # This method is duplicated in DataController and ForceParticle # using the 'times' variables above. Will be useful in those other # locations for particles released at different times # i.e. released over a few days modelTimestep, self.datetimes = AsaTransport.get_time_objects_from_model_timesteps( times, start=self.start) time_chunk = self._time_chunk horiz_chunk = self._horiz_chunk low_memory = kwargs.get("low_memory", False) # Should we remove the cache file at the end of the run? remove_cache = kwargs.get("remove_cache", True) self.bathy_path = kwargs.get("bathy", None) self.cache_path = kwargs.get("cache", None) if self.cache_path is None: # Generate temp filename for dataset cache default_cache_dir = os.path.join(os.path.dirname(__file__), "_cache") temp_name = AsaRandom.filename(prefix=str( datetime.now().microsecond), suffix=".nc") self.cache_path = os.path.join(default_cache_dir, temp_name) logger.progress((1, "Setting up particle start locations")) point_locations = [] if isinstance(self.geometry, Point): point_locations = [self.reference_location] * self._npart elif isinstance(self.geometry, Polygon) or isinstance( self.geometry, MultiPolygon): point_locations = [ Location4D(latitude=loc.y, longitude=loc.x, depth=self._depth, time=self.start) for loc in AsaTransport.fill_polygon_with_points( goal=self._npart, polygon=self.geometry) ] # Initialize the particles logger.progress((2, "Initializing particles")) for x in xrange(0, self._npart): p = LarvaParticle(id=x) p.location = point_locations[x] # We don't need to fill the location gaps here for environment variables # because the first data collected actually relates to this original # position. # We do need to fill in fields such as settled, halted, etc. p.fill_status_gap() # Set the inital note p.note = p.outputstring() p.notes.append(p.note) self.particles.append(p) # This is where it makes sense to implement the multiprocessing # looping for particles and models. Can handle each particle in # parallel probably. # # Get the number of cores (may take some tuning) and create that # many workers then pass particles into the queue for the workers mgr = multiprocessing.Manager() nproc = multiprocessing.cpu_count() - 1 if nproc <= 0: raise ValueError( "Model does not run using less than two CPU cores") # Each particle is a task, plus the DataController number_of_tasks = len(self.particles) + 1 # We need a process for each particle and one for the data controller nproc = min(number_of_tasks, nproc) # When a particle requests data data_request_lock = mgr.Lock() # PID of process with lock has_data_request_lock = mgr.Value('int', -1) nproc_lock = mgr.Lock() # Create the task queue for all of the particles and the DataController tasks = multiprocessing.JoinableQueue(number_of_tasks) # Create the result queue for all of the particles and the DataController results = mgr.Queue(number_of_tasks) # Create the shared state objects get_data = mgr.Value('bool', True) # Number of tasks n_run = mgr.Value('int', number_of_tasks) updating = mgr.Value('bool', False) # When something is reading from cache file read_lock = mgr.Lock() # list of PIDs that are reading has_read_lock = mgr.list() read_count = mgr.Value('int', 0) # When something is writing to the cache file write_lock = mgr.Lock() # PID of process with lock has_write_lock = mgr.Value('int', -1) point_get = mgr.Value('list', [0, 0, 0]) active = mgr.Value('bool', True) logger.progress((3, "Initializing and caching hydro model's grid")) try: ds = CommonDataset.open(hydrodataset) # Query the dataset for common variable names # and the time variable. logger.debug("Retrieving variable information from dataset") common_variables = self.get_common_variables_from_dataset(ds) logger.debug("Pickling time variable to disk for particles") timevar = ds.gettimevar(common_variables.get("u")) f, timevar_pickle_path = tempfile.mkstemp() os.close(f) f = open(timevar_pickle_path, "wb") pickle.dump(timevar, f) f.close() ds.closenc() except: logger.warn("Failed to access remote dataset %s" % hydrodataset) raise DataControllerError("Inaccessible DAP endpoint: %s" % hydrodataset) # Add data controller to the queue first so that it # can get the initial data and is not blocked logger.debug('Starting DataController') logger.progress((4, "Starting processes")) data_controller = parallel.DataController(hydrodataset, common_variables, n_run, get_data, write_lock, has_write_lock, read_lock, read_count, time_chunk, horiz_chunk, times, self.start, point_get, self.reference_location, low_memory=low_memory, cache=self.cache_path) tasks.put(data_controller) # Create DataController worker data_controller_process = parallel.Consumer(tasks, results, n_run, nproc_lock, active, get_data, name="DataController") data_controller_process.start() logger.debug('Adding %i particles as tasks' % len(self.particles)) for part in self.particles: forcing = parallel.ForceParticle( part, hydrodataset, common_variables, timevar_pickle_path, times, self.start, self._models, self.reference_location.point, self._use_bathymetry, self._use_shoreline, self._use_seasurface, get_data, n_run, read_lock, has_read_lock, read_count, point_get, data_request_lock, has_data_request_lock, reverse_distance=self.reverse_distance, bathy=self.bathy_path, shoreline_path=self.shoreline_path, cache=self.cache_path, time_method=self.time_method) tasks.put(forcing) # Create workers for the particles. procs = [ parallel.Consumer(tasks, results, n_run, nproc_lock, active, get_data, name="ForceParticle-%d" % i) for i in xrange(nproc - 1) ] for w in procs: w.start() logger.debug('Started %s' % w.name) # Get results back from queue, test for failed particles return_particles = [] retrieved = 0. error_code = 0 logger.info("Waiting for %i particle results" % len(self.particles)) logger.progress((5, "Running model")) while retrieved < number_of_tasks: try: # Returns a tuple of code, result code, tempres = results.get(timeout=240) except Queue.Empty: # Poll the active processes to make sure they are all alive and then continue with loop if not data_controller_process.is_alive( ) and data_controller_process.exitcode != 0: # Data controller is zombied, kill off other processes. get_data.value == False results.put((-2, "DataController")) new_procs = [] old_procs = [] for p in procs: if not p.is_alive() and p.exitcode != 0: # Do what the Consumer would do if something finished. # Add something to results queue results.put((-3, "ZombieParticle")) # Decrement nproc (DataController exits when this is 0) with nproc_lock: n_run.value = n_run.value - 1 # Remove task from queue (so they can be joined later on) tasks.task_done() # Start a new Consumer. It will exit if there are no tasks available. np = parallel.Consumer(tasks, results, n_run, nproc_lock, active, get_data, name=p.name) new_procs.append(np) old_procs.append(p) # Release any locks the PID had if p.pid in has_read_lock: with read_lock: read_count.value -= 1 has_read_lock.remove(p.pid) if has_data_request_lock.value == p.pid: has_data_request_lock.value = -1 try: data_request_lock.release() except: pass if has_write_lock.value == p.pid: has_write_lock.value = -1 try: write_lock.release() except: pass for p in old_procs: try: procs.remove(p) except ValueError: logger.warn( "Did not find %s in the list of processes. Continuing on." % p.name) for p in new_procs: procs.append(p) logger.warn( "Started a new consumer (%s) to replace a zombie consumer" % p.name) p.start() else: # We got one. retrieved += 1 if code == None: logger.warn("Got an unrecognized response from a task.") elif code == -1: logger.warn("Particle %s has FAILED!!" % tempres.uid) elif code == -2: error_code = code logger.warn( "DataController has FAILED!! Removing cache file so the particles fail." ) try: os.remove(self.cache_path) except OSError: logger.debug( "Could not remove cache file, it probably never existed" ) pass elif code == -3: error_code = code logger.info( "A zombie process was caught and task was removed from queue" ) elif isinstance(tempres, Particle): logger.info("Particle %d finished" % tempres.uid) return_particles.append(tempres) # We mulitply by 95 here to save 5% for the exporting logger.progress( (round((retrieved / number_of_tasks) * 90., 1), "Particle %d finished" % tempres.uid)) elif tempres == "DataController": logger.info("DataController finished") logger.progress((round((retrieved / number_of_tasks) * 90., 1), "DataController finished")) else: logger.info("Got a strange result on results queue") logger.info(str(tempres)) logger.info("Retrieved %i/%i results" % (int(retrieved), number_of_tasks)) if len(return_particles) != len(self.particles): logger.warn( "Some particles failed and are not included in the output") # The results queue should be empty at this point assert results.empty() is True # Should be good to join on the tasks now that the queue is empty logger.info("Joining the task queue") tasks.join() # Join all processes logger.info("Joining the processes") for w in procs + [data_controller_process]: # Wait 10 seconds w.join(10.) if w.is_alive(): # Process is hanging, kill it. logger.info( "Terminating %s forcefully. This should have exited itself." % w.name) w.terminate() logger.info('Workers complete') self.particles = return_particles # Remove Manager so it shuts down del mgr # Remove pickled timevar os.remove(timevar_pickle_path) # Remove the cache file if remove_cache is True: try: os.remove(self.cache_path) except OSError: logger.debug( "Could not remove cache file, it probably never existed") logger.progress((96, "Exporting results")) if len(self.particles) > 0: # If output_formats and path specified, # output particle run data to disk when completed if "output_formats" in kwargs: # Make sure output_path is also included if kwargs.get("output_path", None) != None: formats = kwargs.get("output_formats") output_path = kwargs.get("output_path") if isinstance(formats, list): for format in formats: logger.info("Exporting to: %s" % format) try: self.export(output_path, format=format) except: logger.error("Failed to export to: %s" % format) else: logger.warn( 'The output_formats parameter should be a list, not saving any output!' ) else: logger.warn( 'No output path defined, not saving any output!') else: logger.warn('No output format defined, not saving any output!') else: logger.warn("Model didn't actually do anything, check the log.") if error_code == -2: raise DataControllerError("Error in the DataController") else: raise ModelError("Error in the model") logger.progress((99, "Model Run Complete")) return
def run(self, hydrodataset, **kwargs): # Add ModelController description to logfile logger.info(self) # Add the model descriptions to logfile for m in self._models: logger.info(m) # Calculate the model timesteps # We need times = len(self._nstep) + 1 since data is stored one timestep # after a particle is forced with the final timestep's data. times = range(0,(self._step*self._nstep)+1,self._step) # Calculate a datetime object for each model timestep # This method is duplicated in DataController and ForceParticle # using the 'times' variables above. Will be useful in those other # locations for particles released at different times # i.e. released over a few days modelTimestep, self.datetimes = AsaTransport.get_time_objects_from_model_timesteps(times, start=self.start) time_chunk = self._time_chunk horiz_chunk = self._horiz_chunk low_memory = kwargs.get("low_memory", False) # Should we remove the cache file at the end of the run? remove_cache = kwargs.get("remove_cache", True) self.bathy_path = kwargs.get("bathy", None) self.cache_path = kwargs.get("cache", None) if self.cache_path is None: # Generate temp filename for dataset cache default_cache_dir = os.path.join(os.path.dirname(__file__), "_cache") temp_name = AsaRandom.filename(prefix=str(datetime.now().microsecond), suffix=".nc") self.cache_path = os.path.join(default_cache_dir, temp_name) logger.progress((1, "Setting up particle start locations")) point_locations = [] if isinstance(self.geometry, Point): point_locations = [self.reference_location] * self._npart elif isinstance(self.geometry, Polygon) or isinstance(self.geometry, MultiPolygon): point_locations = [Location4D(latitude=loc.y, longitude=loc.x, depth=self._depth, time=self.start) for loc in AsaTransport.fill_polygon_with_points(goal=self._npart, polygon=self.geometry)] # Initialize the particles logger.progress((2, "Initializing particles")) for x in xrange(0, self._npart): p = LarvaParticle(id=x) p.location = point_locations[x] # We don't need to fill the location gaps here for environment variables # because the first data collected actually relates to this original # position. # We do need to fill in fields such as settled, halted, etc. p.fill_status_gap() # Set the inital note p.note = p.outputstring() p.notes.append(p.note) self.particles.append(p) # This is where it makes sense to implement the multiprocessing # looping for particles and models. Can handle each particle in # parallel probably. # # Get the number of cores (may take some tuning) and create that # many workers then pass particles into the queue for the workers mgr = multiprocessing.Manager() nproc = multiprocessing.cpu_count() - 1 if nproc <= 0: raise ValueError("Model does not run using less than two CPU cores") # Each particle is a task, plus the DataController number_of_tasks = len(self.particles) + 1 # We need a process for each particle and one for the data controller nproc = min(number_of_tasks, nproc) # When a particle requests data data_request_lock = mgr.Lock() # PID of process with lock has_data_request_lock = mgr.Value('int',-1) nproc_lock = mgr.Lock() # Create the task queue for all of the particles and the DataController tasks = multiprocessing.JoinableQueue(number_of_tasks) # Create the result queue for all of the particles and the DataController results = mgr.Queue(number_of_tasks) # Create the shared state objects get_data = mgr.Value('bool', True) # Number of tasks n_run = mgr.Value('int', number_of_tasks) updating = mgr.Value('bool', False) # When something is reading from cache file read_lock = mgr.Lock() # list of PIDs that are reading has_read_lock = mgr.list() read_count = mgr.Value('int', 0) # When something is writing to the cache file write_lock = mgr.Lock() # PID of process with lock has_write_lock = mgr.Value('int',-1) point_get = mgr.Value('list', [0, 0, 0]) active = mgr.Value('bool', True) logger.progress((3, "Initializing and caching hydro model's grid")) try: ds = CommonDataset.open(hydrodataset) # Query the dataset for common variable names # and the time variable. logger.debug("Retrieving variable information from dataset") common_variables = self.get_common_variables_from_dataset(ds) logger.debug("Pickling time variable to disk for particles") timevar = ds.gettimevar(common_variables.get("u")) f, timevar_pickle_path = tempfile.mkstemp() os.close(f) f = open(timevar_pickle_path, "wb") pickle.dump(timevar, f) f.close() ds.closenc() except: logger.warn("Failed to access remote dataset %s" % hydrodataset) raise DataControllerError("Inaccessible DAP endpoint: %s" % hydrodataset) # Add data controller to the queue first so that it # can get the initial data and is not blocked logger.debug('Starting DataController') logger.progress((4, "Starting processes")) data_controller = parallel.DataController(hydrodataset, common_variables, n_run, get_data, write_lock, has_write_lock, read_lock, read_count, time_chunk, horiz_chunk, times, self.start, point_get, self.reference_location, low_memory=low_memory, cache=self.cache_path) tasks.put(data_controller) # Create DataController worker data_controller_process = parallel.Consumer(tasks, results, n_run, nproc_lock, active, get_data, name="DataController") data_controller_process.start() logger.debug('Adding %i particles as tasks' % len(self.particles)) for part in self.particles: forcing = parallel.ForceParticle(part, hydrodataset, common_variables, timevar_pickle_path, times, self.start, self._models, self.reference_location.point, self._use_bathymetry, self._use_shoreline, self._use_seasurface, get_data, n_run, read_lock, has_read_lock, read_count, point_get, data_request_lock, has_data_request_lock, reverse_distance=self.reverse_distance, bathy=self.bathy_path, shoreline_path=self.shoreline_path, shoreline_feature=self.shoreline_feature, cache=self.cache_path, time_method=self.time_method) tasks.put(forcing) # Create workers for the particles. procs = [ parallel.Consumer(tasks, results, n_run, nproc_lock, active, get_data, name="ForceParticle-%d"%i) for i in xrange(nproc - 1) ] for w in procs: w.start() logger.debug('Started %s' % w.name) # Get results back from queue, test for failed particles return_particles = [] retrieved = 0. error_code = 0 logger.info("Waiting for %i particle results" % len(self.particles)) logger.progress((5, "Running model")) while retrieved < number_of_tasks: try: # Returns a tuple of code, result code, tempres = results.get(timeout=240) except Queue.Empty: # Poll the active processes to make sure they are all alive and then continue with loop if not data_controller_process.is_alive() and data_controller_process.exitcode != 0: # Data controller is zombied, kill off other processes. get_data.value == False results.put((-2, "DataController")) new_procs = [] old_procs = [] for p in procs: if not p.is_alive() and p.exitcode != 0: # Do what the Consumer would do if something finished. # Add something to results queue results.put((-3, "ZombieParticle")) # Decrement nproc (DataController exits when this is 0) with nproc_lock: n_run.value = n_run.value - 1 # Remove task from queue (so they can be joined later on) tasks.task_done() # Start a new Consumer. It will exit if there are no tasks available. np = parallel.Consumer(tasks, results, n_run, nproc_lock, active, get_data, name=p.name) new_procs.append(np) old_procs.append(p) # Release any locks the PID had if p.pid in has_read_lock: with read_lock: read_count.value -= 1 has_read_lock.remove(p.pid) if has_data_request_lock.value == p.pid: has_data_request_lock.value = -1 try: data_request_lock.release() except: pass if has_write_lock.value == p.pid: has_write_lock.value = -1 try: write_lock.release() except: pass for p in old_procs: try: procs.remove(p) except ValueError: logger.warn("Did not find %s in the list of processes. Continuing on." % p.name) for p in new_procs: procs.append(p) logger.warn("Started a new consumer (%s) to replace a zombie consumer" % p.name) p.start() else: # We got one. retrieved += 1 if code == None: logger.warn("Got an unrecognized response from a task.") elif code == -1: logger.warn("Particle %s has FAILED!!" % tempres.uid) elif code == -2: error_code = code logger.warn("DataController has FAILED!! Removing cache file so the particles fail.") try: os.remove(self.cache_path) except OSError: logger.debug("Could not remove cache file, it probably never existed") pass elif code == -3: error_code = code logger.info("A zombie process was caught and task was removed from queue") elif isinstance(tempres, Particle): logger.info("Particle %d finished" % tempres.uid) return_particles.append(tempres) # We mulitply by 95 here to save 5% for the exporting logger.progress((round((retrieved / number_of_tasks) * 90.,1), "Particle %d finished" % tempres.uid)) elif tempres == "DataController": logger.info("DataController finished") logger.progress((round((retrieved / number_of_tasks) * 90.,1), "DataController finished")) else: logger.info("Got a strange result on results queue") logger.info(str(tempres)) logger.info("Retrieved %i/%i results" % (int(retrieved),number_of_tasks)) if len(return_particles) != len(self.particles): logger.warn("Some particles failed and are not included in the output") # The results queue should be empty at this point assert results.empty() is True # Should be good to join on the tasks now that the queue is empty logger.info("Joining the task queue") tasks.join() # Join all processes logger.info("Joining the processes") for w in procs + [data_controller_process]: # Wait 10 seconds w.join(10.) if w.is_alive(): # Process is hanging, kill it. logger.info("Terminating %s forcefully. This should have exited itself." % w.name) w.terminate() logger.info('Workers complete') self.particles = return_particles # Remove Manager so it shuts down del mgr # Remove pickled timevar os.remove(timevar_pickle_path) # Remove the cache file if remove_cache is True: try: os.remove(self.cache_path) except OSError: logger.debug("Could not remove cache file, it probably never existed") logger.progress((96, "Exporting results")) if len(self.particles) > 0: # If output_formats and path specified, # output particle run data to disk when completed if "output_formats" in kwargs: # Make sure output_path is also included if kwargs.get("output_path", None) != None: formats = kwargs.get("output_formats") output_path = kwargs.get("output_path") if isinstance(formats, list): for format in formats: logger.info("Exporting to: %s" % format) try: self.export(output_path, format=format) except: logger.error("Failed to export to: %s" % format) else: logger.warn('The output_formats parameter should be a list, not saving any output!') else: logger.warn('No output path defined, not saving any output!') else: logger.warn('No output format defined, not saving any output!') else: logger.warn("Model didn't actually do anything, check the log.") if error_code == -2: raise DataControllerError("Error in the DataController") else: raise ModelError("Error in the model") logger.progress((99, "Model Run Complete")) return
def __reverse(self, **kwargs): """ Reverse particle just off of the shore in the direction that it came in. Adds a slight random factor to the distance and angle it is reversed in. """ start_point = kwargs.pop('start_point') hit_point = kwargs.pop('hit_point') distance = kwargs.pop('distance') azimuth = kwargs.pop('azimuth') reverse_azimuth = kwargs.pop('reverse_azimuth') reverse_distance = kwargs.get('reverse_distance', None) if reverse_distance is None: reverse_distance = 100 # Randomize the reverse angle slightly (+/- 5 degrees) random_azimuth = reverse_azimuth + AsaRandom.random() * 5 count = 0 nudge_distance = 0.01 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) # Find point just offshore to do testing with. Try 15 times (~350m). This makes sure the start_point is in the water # for the next call to intersect (next while loop). while self.intersect(single_point=nudge_loc.point) and count < 16: nudge_distance *= 2 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) count += 1 # We tried 16 times and couldn't find a point. This should totally never happen. if count == 16: logger.debug( "WOW. Could not find location in water to do shoreline calculation with. Assuming particle did not move from original location" ) return start_point # Keep trying to throw particle back, halfing the distance each time until it is in water. # Only half it 12 times before giving up and returning the point which the particle came from. count = 0 # Distance amount to half each iteration changing_distance = reverse_distance new_point = AsaGreatCircle.great_circle(distance=reverse_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) while self.intersect(start_point=nudge_loc.point, end_point=new_loc.point) and count < 12: changing_distance /= 2 new_point = AsaGreatCircle.great_circle(distance=changing_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) count += 1 # We tried 10 times and the particle was still on shore, return the point the particle started from. # No randomization. if count == 12: logger.debug( "Could not react particle with shoreline. Assuming particle did not move from original location" ) return start_point return new_loc
def setup_run(self, hydrodataset, **kwargs): super(CachingModelController, self).setup_run(hydrodataset, **kwargs) # Get the number of cores (may take some tuning) and create that # many workers then pass particles into the queue for the workers self.mgr = multiprocessing.Manager() # This tracks if the system is 'alive'. Most looping whiles will check this # and break out if it is False. This is True until something goes very wrong. self.active = self.mgr.Value('bool', True) # Either spin up the number of cores, or the number of tasks self.nproc = min(multiprocessing.cpu_count() - 1, self.total_task_count()) # Number of tasks that we need to run. This is decremented everytime something exits. self.n_run = self.mgr.Value('int', self.total_task_count()) # The lock that controls access to the 'n_run' variable self.nproc_lock = self.mgr.Lock() # Create the task queue for all of the particles and the CachingDataController self.tasks = multiprocessing.JoinableQueue(self.total_task_count()) # Create the result queue for all of the particles and the CachingDataController self.results = self.mgr.Queue(self.total_task_count()) # Should we remove the cache file at the end of the run? self.remove_cache = kwargs.get("remove_cache", False) self.cache_path = kwargs.get("cache_path", None) # Create a temp file for the cache if nothing was passed in if self.cache_path is None: default_cache_dir = os.path.join(os.path.dirname(__file__), "_cache") temp_name = AsaRandom.filename(prefix=str( datetime.now().microsecond), suffix=".nc") self.cache_path = os.path.join(default_cache_dir, temp_name) # Be sure the cache directory exists if not os.path.exists(os.path.dirname(self.cache_path)): logger.info("Creating cache directory: %s" % self.cache_path) os.makedirs(os.path.dirname(self.cache_path)) # Create the shared state objects # Particles use this to tell the Data Controller to "get_data". # The CachingDataController sets this to False when it is done writing to the cache file. # Particles will wait for this to be False before reading from the cache file. # If we are caching, this starts as True so the Particles don't take off. If we # are not caching, this is False so the Particles can start immediatly. self.get_data = self.mgr.Value('bool', True) # Particles use this to tell the DataContoller which indices to 'get_data' for self.point_get = self.mgr.Value('list', [0, 0, 0]) # This locks access to the 'has_data_request_lock' value self.data_request_lock = self.mgr.Lock() # This tracks which Particle PID is asking the CachingDataController for data self.has_data_request_lock = self.mgr.Value('int', -1) # The lock that controls access to modifying 'has_read_lock' and 'read_count' self.read_lock = self.mgr.Lock() # List of Particle PIDs that are reading from the cache self.has_read_lock = self.mgr.list() # The number of Particles that are reading from the cache self.read_count = self.mgr.Value('int', 0) # When something is writing to the cache file self.write_lock = self.mgr.Lock() # PID of process with lock self.has_write_lock = self.mgr.Value('int', -1)
def setup_run(self, **kwargs): super(CachingModelController, self).setup_run(**kwargs) # Get the number of cores (may take some tuning) and create that # many workers then pass particles into the queue for the workers self.mgr = multiprocessing.Manager() # This tracks if the system is 'alive'. Most looping whiles will check this # and break out if it is False. This is True until something goes very wrong. self.active = self.mgr.Value('bool', True) # Either spin up the number of cores, or the number of tasks self.nproc = min(multiprocessing.cpu_count() - 1, self.number_of_tasks) # Number of tasks that we need to run. This is decremented everytime something exits. self.n_run = self.mgr.Value('int', self.number_of_tasks) # The lock that controls access to the 'n_run' variable self.nproc_lock = self.mgr.Lock() # Create the task queue for all of the particles and the CachingDataController self.tasks = multiprocessing.JoinableQueue(self.number_of_tasks) # Create the result queue for all of the particles and the CachingDataController self.results = self.mgr.Queue(self.number_of_tasks) # Should we remove the cache file at the end of the run? self.remove_cache = kwargs.get("remove_cache", False) self.cache_path = kwargs.get("cache_path", None) # Create a temp file for the cache if nothing was passed in if self.cache_path is None: default_cache_dir = os.path.join(os.path.dirname(__file__), "_cache") temp_name = AsaRandom.filename(prefix=str(datetime.now().microsecond), suffix=".nc") self.cache_path = os.path.join(default_cache_dir, temp_name) # Be sure the cache directory exists if not os.path.exists(os.path.dirname(self.cache_path)): logger.info("Creating cache directory: %s" % self.cache_path) os.makedirs(os.path.dirname(self.cache_path)) # Create the shared state objects # Particles use this to tell the Data Controller to "get_data". # The CachingDataController sets this to False when it is done writing to the cache file. # Particles will wait for this to be False before reading from the cache file. # If we are caching, this starts as True so the Particles don't take off. If we # are not caching, this is False so the Particles can start immediatly. self.get_data = self.mgr.Value('bool', True) # Particles use this to tell the DataContoller which indices to 'get_data' for self.point_get = self.mgr.Value('list', [0, 0, 0]) # This locks access to the 'has_data_request_lock' value self.data_request_lock = self.mgr.Lock() # This tracks which Particle PID is asking the CachingDataController for data self.has_data_request_lock = self.mgr.Value('int', -1) # The lock that controls access to modifying 'has_read_lock' and 'read_count' self.read_lock = self.mgr.Lock() # List of Particle PIDs that are reading from the cache self.has_read_lock = self.mgr.list() # The number of Particles that are reading from the cache self.read_count = self.mgr.Value('int', 0) # When something is writing to the cache file self.write_lock = self.mgr.Lock() # PID of process with lock self.has_write_lock = self.mgr.Value('int', -1)
def __reverse(self, **kwargs): """ Reverse particle just off of the shore in the direction that it came in. Adds a slight random factor to the distance and angle it is reversed in. """ #st = time.clock() start_point = kwargs.pop('start_point') hit_point = kwargs.pop('hit_point') reverse_azimuth = kwargs.pop('reverse_azimuth') reverse_distance = kwargs.get('reverse_distance', None) if reverse_distance is None: reverse_distance = 100 # Randomize the reverse angle slightly (+/- 5 degrees) random_azimuth = reverse_azimuth + AsaRandom.random() * 5 count = 0 nudge_distance = 0.01 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) # Find point just offshore to do testing with. Try 15 times (~350m). This makes sure the start_point is in the water # for the next call to intersect (next while loop). while self.intersect(single_point=nudge_loc.point) and count < 16: nudge_distance *= 2 nudge_point = AsaGreatCircle.great_circle(distance=nudge_distance, azimuth=reverse_azimuth, start_point=hit_point) nudge_loc = Location4D(latitude=nudge_point['latitude'], longitude=nudge_point['longitude'], depth=start_point.depth) count += 1 # We tried 16 times and couldn't find a point. This should totally never happen. if count == 16: logger.warn( "LOOK: Could not find location in water to do shoreline calculation with. Assuming particle did not move from original location" ) return start_point # Keep trying to throw particle back, halfing the distance each time until it is in water. # Only half it 6 times before giving up and returning the point which the particle came from. count = 0 # Distance amount to half each iteration changing_distance = reverse_distance new_point = AsaGreatCircle.great_circle(distance=reverse_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) # We don't want to reverse further than the current spatial buffer, because we will reindex the # source file everytime we reverse, which will slow down the calculations considerably. while (not self._spatial_query_object.contains(new_loc.point) or self.intersect(start_point=nudge_loc.point, end_point=new_loc.point)) and count < 6: changing_distance /= 2 new_point = AsaGreatCircle.great_circle(distance=changing_distance, azimuth=random_azimuth, start_point=hit_point) new_loc = Location4D(latitude=new_point['latitude'], longitude=new_point['longitude'], depth=start_point.depth) count += 1 # We tried 10 times and the particle was still on shore, return the point the particle started from. # No randomization. if count == 6: logger.warn( "LOOK: Could not react particle with shoreline. Assuming particle did not move from original location" ) return start_point #logger.info("Reaction time: %f" % (time.clock() - st)) return new_loc