def test_diel_migration(self):
num_steps = 168
num_particles = 4
start_time = datetime(2013, 4, 1, 0, tzinfo=pytz.utc)
# Behavior
behavior_config = open(os.path.normpath(os.path.join(os.path.dirname(__file__), "./resources/files/diel_suncycles.json"))).read()
lb = LarvaBehavior(json=behavior_config)
models = [self.transport]
models.append(lb)
model = CachingModelController(
latitude=60.68,
longitude=-146.42,
depth=self.start_depth,
start=start_time,
step=self.time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
use_shoreline=True,
time_chunk=24,
horiz_chunk=2,
time_method='nearest',
bathy_path=self.bathy_file)
model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_DAS.nc", cache=self.cache_path)
model.run(output_path=self.output_path, output_formats=self.output_formats)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
def test_quick_settlement(self):
num_steps = 24
num_particles = 4
# Behavior
behavior_config = open(os.path.normpath(os.path.join(os.path.dirname(__file__), "./resources/files/behavior_quick_settle.json"))).read()
lb = LarvaBehavior(json=behavior_config)
models = [self.transport]
models.append(lb)
model = CachingModelController(
latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
use_shoreline=True,
time_chunk=12,
horiz_chunk=2,
time_method='nearest',
bathy_path=self.bathy_file)
model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_DAS.nc", cache=self.cache_path)
model.run(output_path=self.output_path, output_formats=self.output_formats)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
def test_no_dap_data_for_requested_run(self):
models = [self.transport]
# Start is after available time
model = CachingModelController(latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=datetime.utcnow() +
timedelta(days=30),
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False)
with self.assertRaises(BaseDataControllerError):
model.setup_run(
"http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
# Start is OK but Ending is after available time
model = CachingModelController(latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=datetime.utcnow() -
timedelta(days=2),
step=self.time_step,
nstep=500,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False)
with self.assertRaises(BaseDataControllerError):
model.setup_run(
"http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
def test_sheep_bay(self):
self.log.logger.info("**************************************")
self.log.logger.info("Running: Sheep Bay")
# 6 days
num_steps = 1632
num_particles = 100
time_step = 3600
behavior_config = json.loads(urlopen("http://behaviors.larvamap.asascience.com/library/50ef1bb1cc7b61000700001d.json").read())
lb = LarvaBehavior(data=behavior_config[u'results'][0])
models = [Transport(horizDisp=0.01, vertDisp=0.001)]
models.append(lb)
start_time = datetime(2011, 5, 2, 00, tzinfo=pytz.utc)
start_lat = 60.60899655733162
start_lon = -145.97402533055956
depth = -1
shoreline_path = os.path.join(self.shoreline_path, "westcoast", "New_Land_Clean.shp")
model = CachingModelController(
latitude=start_lat,
longitude=start_lon,
depth=depth,
start=start_time,
step=time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
use_shoreline=True,
time_chunk=24,
horiz_chunk=5,
time_method='interp',
shoreline_path=shoreline_path,
bathy_path=self.bathy_file)
model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc", cache=self.cache_path)
model.run(output_path=self.output_path, output_formats=self.output_formats)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
def test_run_from_dap_without_cache(self):
models = [self.transport]
p = Point(self.start_lon, self.start_lat)
model = CachingModelController(geometry=p,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False)
model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
model.run(output_formats=self.output_formats, output_path=self.output_path)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
# We didn't pass remove_cache=False, so it should have been removed by the CachingModelController.
self.assertFalse(os.path.exists(self.cache_path))
def test_no_dap_data_for_requested_run(self):
models = [self.transport]
# Start is after available time
model = CachingModelController(latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=datetime.utcnow() + timedelta(days=30),
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False)
with self.assertRaises(BaseDataControllerError):
model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
# Start is OK but Ending is after available time
model = CachingModelController(latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=datetime.utcnow() - timedelta(days=2),
step=self.time_step,
nstep=500,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False)
with self.assertRaises(BaseDataControllerError):
model.setup_run("http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
def test_run_from_multiple_files_with_cache(self):
models = [self.transport]
p = Point(self.start_lon, self.start_lat)
model = CachingModelController(geometry=p,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
se_shoreline=False,
time_chunk=10,
horiz_chunk=4)
model.setup_run("/data/lm/tests/pws_das_2014*.nc",
cache_path=self.cache_path,
remove_cache=False)
model.run(output_formats=self.output_formats,
output_path=self.output_path)
self.assertTrue(
os.path.exists(
os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(
os.path.join(self.output_path, "simple_trackline.geojson"))
self.assertTrue(os.path.exists(self.cache_path))
os.remove(self.cache_path)
def test_timechunk_greater_than_timestep(self):
models = [self.transport]
model = CachingModelController(latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False,
time_chunk=48,
horiz_chunk=2)
model.setup_run("/data/lm/tests/pws_das_2014*.nc",
cache_path=self.cache_path,
remove_cache=False)
model.run(output_formats=self.output_formats,
output_path=self.output_path)
self.assertTrue(
os.path.exists(
os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(
os.path.join(self.output_path, "simple_trackline.geojson"))
self.assertTrue(os.path.exists(self.cache_path))
os.remove(self.cache_path)
def test_run_from_dap_without_cache(self):
models = [self.transport]
p = Point(self.start_lon, self.start_lat)
model = CachingModelController(geometry=p,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False)
model.setup_run(
"http://thredds.axiomalaska.com/thredds/dodsC/PWS_L2_FCST.nc")
model.run(output_formats=self.output_formats,
output_path=self.output_path)
self.assertTrue(
os.path.exists(
os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(
os.path.join(self.output_path, "simple_trackline.geojson"))
# We didn't pass remove_cache=False, so it should have been removed by the CachingModelController.
self.assertFalse(os.path.exists(self.cache_path))
def test_run_from_multiple_files_with_cache(self):
models = [self.transport]
p = Point(self.start_lon, self.start_lat)
model = CachingModelController(geometry=p,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
se_shoreline=False,
time_chunk=10,
horiz_chunk=4)
model.setup_run("/data/lm/tests/pws_das_2014*.nc", cache_path=self.cache_path, remove_cache=False)
model.run(output_formats=self.output_formats, output_path=self.output_path)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
self.assertTrue(os.path.exists(self.cache_path))
os.remove(self.cache_path)
def test_timechunk_greater_than_timestep(self):
models = [self.transport]
model = CachingModelController(latitude=self.start_lat,
longitude=self.start_lon,
depth=self.start_depth,
start=self.start_time,
step=self.time_step,
nstep=self.num_steps,
npart=self.num_particles,
models=models,
use_bathymetry=False,
use_shoreline=False,
time_chunk=48,
horiz_chunk=2)
model.setup_run("/data/lm/tests/pws_das_2014*.nc", cache_path=self.cache_path, remove_cache=False)
model.run(output_formats=self.output_formats, output_path=self.output_path)
self.assertTrue(os.path.exists(os.path.join(self.output_path, "simple_trackline.geojson")))
self.draw_trackline(os.path.join(self.output_path, "simple_trackline.geojson"))
self.assertTrue(os.path.exists(self.cache_path))
os.remove(self.cache_path)
def run(run_id):
with app.app_context():
job = get_current_job(connection=redis_connection)
output_path = os.path.join(current_app.config['OUTPUT_PATH'], run_id)
shutil.rmtree(output_path, ignore_errors=True)
os.makedirs(output_path)
cache_path = os.path.join(current_app.config['CACHE_PATH'], run_id)
shutil.rmtree(cache_path, ignore_errors=True)
os.makedirs(cache_path)
f, log_file = tempfile.mkstemp(dir=cache_path, prefix=run_id, suffix=".log")
os.close(f)
os.chmod(log_file, 0644)
run = db.Run.find_one( { '_id' : ObjectId(run_id) } )
if run is None:
return "Failed to locate run %s. May have been deleted while task was in the queue?" % run_id
# Wait for PubSub listening to begin
time.sleep(2)
model = None
try:
hydropath = run['hydro_path']
geometry = loads(run['geometry'])
start_depth = run['release_depth']
num_particles = run['particles']
time_step = run['timestep']
num_steps = int(math.ceil((run['duration'] * 24 * 60 * 60) / time_step))
start_time = run['start'].replace(tzinfo = pytz.utc)
shoreline_path = run['shoreline_path'] or app.config.get("SHORE_PATH")
shoreline_feat = run['shoreline_feature']
# Setup Models
models = []
if run['cached_behavior'] is not None and run['cached_behavior'].get('results', None) is not None:
behavior_data = run['cached_behavior']['results'][0]
l = LarvaBehavior(data=behavior_data)
models.append(l)
models.append(Transport(horizDisp=run['horiz_dispersion'], vertDisp=run['vert_dispersion']))
output_formats = [
ex.H5Trackline,
ex.H5TracklineWithPoints,
ex.H5ParticleTracklines,
ex.H5ParticleMultiPoint,
ex.H5GDALShapefile
]
def save_progress(stop, progress_deque, logger):
while True:
if stop():
break
try:
record = progress_deque.pop()
if record == StopIteration:
break
job.meta["updated"] = record[0]
if record is not None and record[1] >= 0:
job.meta["progress"] = record[1]
if isinstance(record[2], unicode) or isinstance(record[2], str):
job.meta["message"] = record[2]
job.save()
except IndexError:
pass
except Exception:
raise
# Relax
time.sleep(0.01)
# Set up Logger
from paegan.logger.progress_handler import ProgressHandler
from paegan.logger.multi_process_logging import MultiProcessingLogHandler
from paegan.logger import logger
# Logging handler
queue = multiprocessing.Queue(-1)
logger.setLevel(logging.PROGRESS)
if app.config.get("DEBUG") is True:
mphandler = MultiProcessingLogHandler(log_file, queue, stream=True)
else:
mphandler = MultiProcessingLogHandler(log_file, queue)
mphandler.setLevel(logging.PROGRESS)
formatter = logging.Formatter('[%(asctime)s] - %(levelname)s - %(name)s - %(processName)s - %(message)s')
mphandler.setFormatter(formatter)
logger.addHandler(mphandler)
# Progress handler
progress_deque = collections.deque(maxlen=1)
progress_handler = ProgressHandler(progress_deque)
progress_handler.setLevel(logging.PROGRESS)
logger.addHandler(progress_handler)
stop_log_listener = False
pl = threading.Thread(name="ProgressUpdater", target=save_progress, args=(lambda: stop_log_listener, progress_deque, logger,))
pl.start()
model = CachingModelController(
geometry=geometry,
depth=start_depth,
start=start_time,
step=time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
bathy_path=current_app.config['BATHY_PATH'],
use_shoreline=True,
time_method=run['time_method'],
time_chunk=run['time_chunk'],
horiz_chunk=run['horiz_chunk'],
shoreline_path=shoreline_path,
shoreline_feature=shoreline_feat,
shoreline_index_buffer=0.05)
cache_file = os.path.join(cache_path, run_id + ".nc.cache")
model.setup_run(hydropath, cache_path=cache_file, remove_cache=False)
run.started = datetime.utcnow()
model.run(output_formats=output_formats, output_path=output_path)
except Exception as e:
# Stop progress log handler
stop_log_listener = True
# Wait for log listener to exit
pl.join()
app.logger.exception("Failed to run model")
job.meta["message"] = e.message
job.meta["outcome"] = "failed"
else:
# Stop progress log handler
stop_log_listener = True
# Wait for log listener to exit
pl.join()
job.meta["message"] = "Complete"
job.meta["outcome"] = "success"
job.meta["progress"] = 100
finally:
# Close and remove the handlers so we can use the log file without a file lock
for hand in list(logger.handlers):
logger.removeHandler(hand)
hand.flush()
hand.close()
queue.close()
queue.join_thread()
time.sleep(1)
# Move logfile to output directory
shutil.move(log_file, os.path.join(output_path, 'model.log'))
# Move cachefile to output directory if we made one
shutil.move(cache_file, os.path.join(output_path, 'hydro_cache.nc'))
output_files = []
for filename in os.listdir(output_path):
outfile = os.path.join(output_path, filename)
output_files.append(outfile)
result_files = []
# Handle results and cleanup
if current_app.config['USE_S3'] is True:
base_access_url = urljoin("http://%s.s3.amazonaws.com/output/" % current_app.config['S3_BUCKET'], run_id)
# Upload results to S3 and remove the local copies
conn = S3Connection()
bucket = conn.get_bucket(current_app.config['S3_BUCKET'])
for outfile in output_files:
# Upload the outfile with the same as the run name
_, ext = os.path.splitext(outfile)
new_filename = slugify(unicode(run['name'])) + ext
k = Key(bucket)
k.key = "output/%s/%s" % (run_id, new_filename)
k.set_contents_from_filename(outfile)
k.set_acl('public-read')
result_files.append(base_access_url + "/" + new_filename)
os.remove(outfile)
shutil.rmtree(output_path, ignore_errors=True)
else:
result_files = output_files
job.meta["updated"] = datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(pytz.utc)
job.save()
# Set output fields
run.output = result_files
run.ended = datetime.utcnow()
run.compute()
run.save()
