def test_diel_migration(self): num_steps = 168 num_particles = 4 start_time = datetime(2013, 4, 1, 0, tzinfo=pytz.utc) # Behavior behavior_config = open(os.path.normpath(os.path.join(os.path.dirname(__file__), "./resources/files/diel_suncycles.json"))).read() lb = LarvaBehavior(json=behavior_config) models = [self.transport] models.append(lb) model = CachingModelController( latitude=60.68, longitude=-146.42, depth=self.start_depth, start=start_time, step=self.time_step, nstep=num_steps, npart=num_particles, models=models, use_bathymetry=True, use_shoreline=True, time_chunk=24, horiz_chunk=2, time_method='nearest', bathy_path=self.bathy_file) model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_DAS.nc", cache=self.cache_path) model.run(output_path=self.output_path, output_formats=self.output_formats) self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
def test_quick_settlement(self): num_steps = 24 num_particles = 4 # Behavior behavior_config = open(os.path.normpath(os.path.join(os.path.dirname(__file__), "./resources/files/behavior_quick_settle.json"))).read() lb = LarvaBehavior(json=behavior_config) models = [self.transport] models.append(lb) model = CachingModelController( latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=num_steps, npart=num_particles, models=models, use_bathymetry=True, use_shoreline=True, time_chunk=12, horiz_chunk=2, time_method='nearest', bathy_path=self.bathy_file) model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_DAS.nc", cache=self.cache_path) model.run(output_path=self.output_path, output_formats=self.output_formats) self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
def test_run_from_multiple_files_with_cache(self): models = [self.transport] p = Point(self.start_lon, self.start_lat) model = CachingModelController(geometry=p, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, se_shoreline=False, time_chunk=10, horiz_chunk=4) model.setup_run("/data/lm/tests/pws_das_2014*.nc", cache_path=self.cache_path, remove_cache=False) model.run(output_formats=self.output_formats, output_path=self.output_path) self.assertTrue( os.path.exists( os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline( os.path.join(self.output_path, "simple_trackline.geojson")) self.assertTrue(os.path.exists(self.cache_path)) os.remove(self.cache_path)
def test_no_dap_data_for_requested_run(self): models = [self.transport] # Start is after available time model = CachingModelController(latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=datetime.utcnow() + timedelta(days=30), step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False) with self.assertRaises(BaseDataControllerError): model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc") # Start is OK but Ending is after available time model = CachingModelController(latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=datetime.utcnow() - timedelta(days=2), step=self.time_step, nstep=500, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False) with self.assertRaises(BaseDataControllerError): model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
def test_no_dap_data_for_requested_run(self): models = [self.transport] # Start is after available time model = CachingModelController(latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=datetime.utcnow() + timedelta(days=30), step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False) with self.assertRaises(BaseDataControllerError): model.setup_run( "http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc") # Start is OK but Ending is after available time model = CachingModelController(latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=datetime.utcnow() - timedelta(days=2), step=self.time_step, nstep=500, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False) with self.assertRaises(BaseDataControllerError): model.setup_run( "http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
def test_timechunk_greater_than_timestep(self): models = [self.transport] model = CachingModelController(latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False, time_chunk=48, horiz_chunk=2) model.setup_run("/data/lm/tests/pws_das_2014*.nc", cache_path=self.cache_path, remove_cache=False) model.run(output_formats=self.output_formats, output_path=self.output_path) self.assertTrue( os.path.exists( os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline( os.path.join(self.output_path, "simple_trackline.geojson")) self.assertTrue(os.path.exists(self.cache_path)) os.remove(self.cache_path)
def test_run_from_dap_without_cache(self): models = [self.transport] p = Point(self.start_lon, self.start_lat) model = CachingModelController(geometry=p, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False) model.setup_run( "http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc") model.run(output_formats=self.output_formats, output_path=self.output_path) self.assertTrue( os.path.exists( os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline( os.path.join(self.output_path, "simple_trackline.geojson")) # We didn't pass remove_cache=False, so it should have been removed by the CachingModelController. self.assertFalse(os.path.exists(self.cache_path))
def test_sheep_bay(self): self.log.logger.info("**************************************") self.log.logger.info("Running: Sheep Bay") # 6 days num_steps = 1632 num_particles = 100 time_step = 3600 behavior_config = json.loads(urlopen("http://behaviors.larvamap.asascience.com/library/50ef1bb1cc7b61000700001d.json").read()) lb = LarvaBehavior(data=behavior_config[u'results'][0]) models = [Transport(horizDisp=0.01, vertDisp=0.001)] models.append(lb) start_time = datetime(2011, 5, 2, 00, tzinfo=pytz.utc) start_lat = 60.60899655733162 start_lon = -145.97402533055956 depth = -1 shoreline_path = os.path.join(self.shoreline_path, "westcoast", "New_Land_Clean.shp") model = CachingModelController( latitude=start_lat, longitude=start_lon, depth=depth, start=start_time, step=time_step, nstep=num_steps, npart=num_particles, models=models, use_bathymetry=True, use_shoreline=True, time_chunk=24, horiz_chunk=5, time_method='interp', shoreline_path=shoreline_path, bathy_path=self.bathy_file) model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc", cache=self.cache_path) model.run(output_path=self.output_path, output_formats=self.output_formats) self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
def test_run_from_dap_without_cache(self): models = [self.transport] p = Point(self.start_lon, self.start_lat) model = CachingModelController(geometry=p, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False) model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc") model.run(output_formats=self.output_formats, output_path=self.output_path) self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson")) # We didn't pass remove_cache=False, so it should have been removed by the CachingModelController. self.assertFalse(os.path.exists(self.cache_path))
def test_run_from_multiple_files_with_cache(self): models = [self.transport] p = Point(self.start_lon, self.start_lat) model = CachingModelController(geometry=p, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, se_shoreline=False, time_chunk=10, horiz_chunk=4) model.setup_run("/data/lm/tests/pws_das_2014*.nc", cache_path=self.cache_path, remove_cache=False) model.run(output_formats=self.output_formats, output_path=self.output_path) self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson")) self.assertTrue(os.path.exists(self.cache_path)) os.remove(self.cache_path)
def test_timechunk_greater_than_timestep(self): models = [self.transport] model = CachingModelController(latitude=self.start_lat, longitude=self.start_lon, depth=self.start_depth, start=self.start_time, step=self.time_step, nstep=self.num_steps, npart=self.num_particles, models=models, use_bathymetry=False, use_shoreline=False, time_chunk=48, horiz_chunk=2) model.setup_run("/data/lm/tests/pws_das_2014*.nc", cache_path=self.cache_path, remove_cache=False) model.run(output_formats=self.output_formats, output_path=self.output_path) self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson"))) self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson")) self.assertTrue(os.path.exists(self.cache_path)) os.remove(self.cache_path)
def run(run_id): with app.app_context(): job = get_current_job(connection=redis_connection) output_path = os.path.join(current_app.config['OUTPUT_PATH'], run_id) shutil.rmtree(output_path, ignore_errors=True) os.makedirs(output_path) cache_path = os.path.join(current_app.config['CACHE_PATH'], run_id) shutil.rmtree(cache_path, ignore_errors=True) os.makedirs(cache_path) f, log_file = tempfile.mkstemp(dir=cache_path, prefix=run_id, suffix=".log") os.close(f) os.chmod(log_file, 0644) run = db.Run.find_one( { '_id' : ObjectId(run_id) } ) if run is None: return "Failed to locate run %s. May have been deleted while task was in the queue?" % run_id # Wait for PubSub listening to begin time.sleep(2) model = None try: hydropath = run['hydro_path'] geometry = loads(run['geometry']) start_depth = run['release_depth'] num_particles = run['particles'] time_step = run['timestep'] num_steps = int(math.ceil((run['duration'] * 24 * 60 * 60) / time_step)) start_time = run['start'].replace(tzinfo = pytz.utc) shoreline_path = run['shoreline_path'] or app.config.get("SHORE_PATH") shoreline_feat = run['shoreline_feature'] # Setup Models models = [] if run['cached_behavior'] is not None and run['cached_behavior'].get('results', None) is not None: behavior_data = run['cached_behavior']['results'][0] l = LarvaBehavior(data=behavior_data) models.append(l) models.append(Transport(horizDisp=run['horiz_dispersion'], vertDisp=run['vert_dispersion'])) output_formats = [ ex.H5Trackline, ex.H5TracklineWithPoints, ex.H5ParticleTracklines, ex.H5ParticleMultiPoint, ex.H5GDALShapefile ] def save_progress(stop, progress_deque, logger): while True: if stop(): break try: record = progress_deque.pop() if record == StopIteration: break job.meta["updated"] = record[0] if record is not None and record[1] >= 0: job.meta["progress"] = record[1] if isinstance(record[2], unicode) or isinstance(record[2], str): job.meta["message"] = record[2] job.save() except IndexError: pass except Exception: raise # Relax time.sleep(0.01) # Set up Logger from paegan.logger.progress_handler import ProgressHandler from paegan.logger.multi_process_logging import MultiProcessingLogHandler from paegan.logger import logger # Logging handler queue = multiprocessing.Queue(-1) logger.setLevel(logging.PROGRESS) if app.config.get("DEBUG") is True: mphandler = MultiProcessingLogHandler(log_file, queue, stream=True) else: mphandler = MultiProcessingLogHandler(log_file, queue) mphandler.setLevel(logging.PROGRESS) formatter = logging.Formatter('[%(asctime)s] - %(levelname)s - %(name)s - %(processName)s - %(message)s') mphandler.setFormatter(formatter) logger.addHandler(mphandler) # Progress handler progress_deque = collections.deque(maxlen=1) progress_handler = ProgressHandler(progress_deque) progress_handler.setLevel(logging.PROGRESS) logger.addHandler(progress_handler) stop_log_listener = False pl = threading.Thread(name="ProgressUpdater", target=save_progress, args=(lambda: stop_log_listener, progress_deque, logger,)) pl.start() model = CachingModelController( geometry=geometry, depth=start_depth, start=start_time, step=time_step, nstep=num_steps, npart=num_particles, models=models, use_bathymetry=True, bathy_path=current_app.config['BATHY_PATH'], use_shoreline=True, time_method=run['time_method'], time_chunk=run['time_chunk'], horiz_chunk=run['horiz_chunk'], shoreline_path=shoreline_path, shoreline_feature=shoreline_feat, shoreline_index_buffer=0.05) cache_file = os.path.join(cache_path, run_id + ".nc.cache") model.setup_run(hydropath, cache_path=cache_file, remove_cache=False) run.started = datetime.utcnow() model.run(output_formats=output_formats, output_path=output_path) except Exception as e: # Stop progress log handler stop_log_listener = True # Wait for log listener to exit pl.join() app.logger.exception("Failed to run model") job.meta["message"] = e.message job.meta["outcome"] = "failed" else: # Stop progress log handler stop_log_listener = True # Wait for log listener to exit pl.join() job.meta["message"] = "Complete" job.meta["outcome"] = "success" job.meta["progress"] = 100 finally: # Close and remove the handlers so we can use the log file without a file lock for hand in list(logger.handlers): logger.removeHandler(hand) hand.flush() hand.close() queue.close() queue.join_thread() time.sleep(1) # Move logfile to output directory shutil.move(log_file, os.path.join(output_path, 'model.log')) # Move cachefile to output directory if we made one shutil.move(cache_file, os.path.join(output_path, 'hydro_cache.nc')) output_files = [] for filename in os.listdir(output_path): outfile = os.path.join(output_path, filename) output_files.append(outfile) result_files = [] # Handle results and cleanup if current_app.config['USE_S3'] is True: base_access_url = urljoin("http://%s.s3.amazonaws.com/output/" % current_app.config['S3_BUCKET'], run_id) # Upload results to S3 and remove the local copies conn = S3Connection() bucket = conn.get_bucket(current_app.config['S3_BUCKET']) for outfile in output_files: # Upload the outfile with the same as the run name _, ext = os.path.splitext(outfile) new_filename = slugify(unicode(run['name'])) + ext k = Key(bucket) k.key = "output/%s/%s" % (run_id, new_filename) k.set_contents_from_filename(outfile) k.set_acl('public-read') result_files.append(base_access_url + "/" + new_filename) os.remove(outfile) shutil.rmtree(output_path, ignore_errors=True) else: result_files = output_files job.meta["updated"] = datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(pytz.utc) job.save() # Set output fields run.output = result_files run.ended = datetime.utcnow() run.compute() run.save()