def get_info(shoreline_id):
with app.app_context():
# Save results back to Run
shoreline = db.Shoreline.find_one( { '_id' : ObjectId(shoreline_id) } )
if shoreline is None:
return "No Shoreline exists to update, aborting update process for ID %s" % shoreline_id
shoreline.get_info()
shoreline.updated = datetime.utcnow()
shoreline.save()
def get_file_key_and_link(self, file_path):
url = urlparse(file_path)
if url.scheme == '':
base_access_url = app.config.get('NON_S3_OUTPUT_URL', None)
if base_access_url is None:
# Serve assets through Flask... not desired!
app.logger.warning("Serving static output through Flask is not desirable! Set the NON_S3_OUTPUT_URL config variable.")
filename = url.path.split("/")[-1]
# We can get a context here to use url_for
with app.app_context():
file_link = url_for("run_output_download", run_id=self._id, filename=filename)
else:
file_link = base_access_url + file_path.replace(app.config.get("OUTPUT_PATH"), "")
else:
file_link = file_path
# Local file
url = urlparse(file_link)
name, ext = os.path.splitext(url.path)
file_type = "Unknown (%s)" % ext
if ext == ".zip":
file_type = "Shapefile"
elif ext == ".nc":
file_type = "NetCDF"
elif ext == ".cache":
file_type = "Forcing Data"
elif ext == ".json":
file_type = "JSON"
elif ext == ".geojson":
if "particle_tracklines" in name:
file_type = "Particle Tracklines (GeoJSON)"
elif "center_trackline" in name:
file_type = "Center Trackline (GeoJSON)"
elif "particle_multipoint" in name:
file_type = "Particle MultiPoint (GeoJSON)"
elif ext == ".avi":
file_type = "Animation"
elif ext == ".log":
file_type = "Logfile"
elif ext == ".h5":
file_type = "HDF5 Data File"
return { file_type : file_link }
def calc(dataset_id):
with app.app_context():
# Save results back to Run
dataset = db.Dataset.find_one( { '_id' : ObjectId(dataset_id) } )
if dataset is None:
return "No Dataset exists to update, aborting update process for ID %s" % dataset_id
dataset.calc()
dataset.updated = datetime.utcnow()
# Poor man's scheduler, until rq supports scheduling
# Sleep for 5 minutes to prevent crashing the DAP servers
time.sleep(300)
# And then do it again
job = dataset_queue.enqueue_call(func=calc, args=(dataset_id,))
dataset.task_id = unicode(job.id)
dataset.save()
return "Scheduled %s (%s)" % (dataset_id, dataset.name)
def run(run_id):
with app.app_context():
job = get_current_job(connection=redis_connection)
output_path = os.path.join(current_app.config['OUTPUT_PATH'], run_id)
shutil.rmtree(output_path, ignore_errors=True)
os.makedirs(output_path)
cache_path = os.path.join(current_app.config['CACHE_PATH'], run_id)
shutil.rmtree(cache_path, ignore_errors=True)
os.makedirs(cache_path)
f, log_file = tempfile.mkstemp(dir=cache_path, prefix=run_id, suffix=".log")
os.close(f)
os.chmod(log_file, 0644)
run = db.Run.find_one( { '_id' : ObjectId(run_id) } )
if run is None:
return "Failed to locate run %s. May have been deleted while task was in the queue?" % run_id
# Wait for PubSub listening to begin
time.sleep(2)
model = None
try:
hydropath = run['hydro_path']
geometry = loads(run['geometry'])
start_depth = run['release_depth']
num_particles = run['particles']
time_step = run['timestep']
num_steps = int(math.ceil((run['duration'] * 24 * 60 * 60) / time_step))
start_time = run['start'].replace(tzinfo = pytz.utc)
shoreline_path = run['shoreline_path'] or app.config.get("SHORE_PATH")
shoreline_feat = run['shoreline_feature']
# Setup Models
models = []
if run['cached_behavior'] is not None and run['cached_behavior'].get('results', None) is not None:
behavior_data = run['cached_behavior']['results'][0]
l = LarvaBehavior(data=behavior_data)
models.append(l)
models.append(Transport(horizDisp=run['horiz_dispersion'], vertDisp=run['vert_dispersion']))
output_formats = [
ex.H5Trackline,
ex.H5TracklineWithPoints,
ex.H5ParticleTracklines,
ex.H5ParticleMultiPoint,
ex.H5GDALShapefile
]
def save_progress(stop, progress_deque, logger):
while True:
if stop():
break
try:
record = progress_deque.pop()
if record == StopIteration:
break
job.meta["updated"] = record[0]
if record is not None and record[1] >= 0:
job.meta["progress"] = record[1]
if isinstance(record[2], unicode) or isinstance(record[2], str):
job.meta["message"] = record[2]
job.save()
except IndexError:
pass
except Exception:
raise
# Relax
time.sleep(0.01)
# Set up Logger
from paegan.logger.progress_handler import ProgressHandler
from paegan.logger.multi_process_logging import MultiProcessingLogHandler
from paegan.logger import logger
# Logging handler
queue = multiprocessing.Queue(-1)
logger.setLevel(logging.PROGRESS)
if app.config.get("DEBUG") is True:
mphandler = MultiProcessingLogHandler(log_file, queue, stream=True)
else:
mphandler = MultiProcessingLogHandler(log_file, queue)
mphandler.setLevel(logging.PROGRESS)
formatter = logging.Formatter('[%(asctime)s] - %(levelname)s - %(name)s - %(processName)s - %(message)s')
mphandler.setFormatter(formatter)
logger.addHandler(mphandler)
# Progress handler
progress_deque = collections.deque(maxlen=1)
progress_handler = ProgressHandler(progress_deque)
progress_handler.setLevel(logging.PROGRESS)
logger.addHandler(progress_handler)
stop_log_listener = False
pl = threading.Thread(name="ProgressUpdater", target=save_progress, args=(lambda: stop_log_listener, progress_deque, logger,))
pl.start()
model = CachingModelController(
geometry=geometry,
depth=start_depth,
start=start_time,
step=time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
bathy_path=current_app.config['BATHY_PATH'],
use_shoreline=True,
time_method=run['time_method'],
time_chunk=run['time_chunk'],
horiz_chunk=run['horiz_chunk'],
shoreline_path=shoreline_path,
shoreline_feature=shoreline_feat,
shoreline_index_buffer=0.05)
cache_file = os.path.join(cache_path, run_id + ".nc.cache")
model.setup_run(hydropath, cache_path=cache_file, remove_cache=False)
run.started = datetime.utcnow()
model.run(output_formats=output_formats, output_path=output_path)
except Exception as e:
# Stop progress log handler
stop_log_listener = True
# Wait for log listener to exit
pl.join()
app.logger.exception("Failed to run model")
job.meta["message"] = e.message
job.meta["outcome"] = "failed"
else:
# Stop progress log handler
stop_log_listener = True
# Wait for log listener to exit
pl.join()
job.meta["message"] = "Complete"
job.meta["outcome"] = "success"
job.meta["progress"] = 100
finally:
# Close and remove the handlers so we can use the log file without a file lock
for hand in list(logger.handlers):
logger.removeHandler(hand)
hand.flush()
hand.close()
queue.close()
queue.join_thread()
time.sleep(1)
# Move logfile to output directory
shutil.move(log_file, os.path.join(output_path, 'model.log'))
# Move cachefile to output directory if we made one
shutil.move(cache_file, os.path.join(output_path, 'hydro_cache.nc'))
output_files = []
for filename in os.listdir(output_path):
outfile = os.path.join(output_path, filename)
output_files.append(outfile)
result_files = []
# Handle results and cleanup
if current_app.config['USE_S3'] is True:
base_access_url = urljoin("http://%s.s3.amazonaws.com/output/" % current_app.config['S3_BUCKET'], run_id)
# Upload results to S3 and remove the local copies
conn = S3Connection()
bucket = conn.get_bucket(current_app.config['S3_BUCKET'])
for outfile in output_files:
# Upload the outfile with the same as the run name
_, ext = os.path.splitext(outfile)
new_filename = slugify(unicode(run['name'])) + ext
k = Key(bucket)
k.key = "output/%s/%s" % (run_id, new_filename)
k.set_contents_from_filename(outfile)
k.set_acl('public-read')
result_files.append(base_access_url + "/" + new_filename)
os.remove(outfile)
shutil.rmtree(output_path, ignore_errors=True)
else:
result_files = output_files
job.meta["updated"] = datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(pytz.utc)
job.save()
# Set output fields
run.output = result_files
run.ended = datetime.utcnow()
run.compute()
run.save()
def run(run_id):
# Sleep to give the Run object enough time to save
time.sleep(10)
with app.app_context():
from paegan.logger import logger
job = get_current_job()
output_path = os.path.join(current_app.config['OUTPUT_PATH'], run_id)
shutil.rmtree(output_path, ignore_errors=True)
os.makedirs(output_path)
cache_path = os.path.join(current_app.config['CACHE_PATH'], run_id)
shutil.rmtree(cache_path, ignore_errors=True)
os.makedirs(cache_path)
temp_animation_path = os.path.join(current_app.config['OUTPUT_PATH'], "temp_images_" + run_id)
shutil.rmtree(temp_animation_path, ignore_errors=True)
os.makedirs(temp_animation_path)
# Set up Logger
queue = multiprocessing.Queue(-1)
f, log_file = tempfile.mkstemp(dir=cache_path, prefix=run_id, suffix=".log")
os.close(f)
# Close any existing handlers
(hand.close() for hand in logger.handlers)
# Remove any existing handlers
logger.handlers = []
logger.setLevel(logging.PROGRESS)
handler = MultiProcessingLogHandler(log_file, queue)
handler.setLevel(logging.PROGRESS)
formatter = logging.Formatter('[%(asctime)s] - %(levelname)s - %(name)s - %(processName)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
# Progress stuff. Hokey!
progress_deque = collections.deque(maxlen=1)
progress_handler = ProgressHandler(progress_deque)
progress_handler.setLevel(logging.PROGRESS)
logger.addHandler(progress_handler)
e = threading.Event()
def save_progress():
while e.wait(5) is not True:
try:
record = progress_deque.pop()
if record == StopIteration:
break
job.meta["updated"] = record[0]
if record is not None and record[1] >= 0:
job.meta["progress"] = record[1]
if isinstance(record[2], unicode) or isinstance(record[2], str):
job.meta["message"] = record[2]
job.save()
except IndexError:
pass
except Exception:
raise
return
t = threading.Thread(name="ProgressUpdater", target=save_progress)
t.daemon = True
t.start()
model = None
try:
logger.progress((0, "Configuring model"))
run = db.Run.find_one( { '_id' : ObjectId(run_id) } )
if run is None:
return "Failed to locate run %s. May have been deleted while task was in the queue?" % run_id
geometry = loads(run['geometry'])
start_depth = run['release_depth']
num_particles = run['particles']
time_step = run['timestep']
num_steps = int(math.ceil((run['duration'] * 24 * 60 * 60) / time_step))
start_time = run['start'].replace(tzinfo = pytz.utc)
shoreline_path = run['shoreline_path'] or app.config.get("SHORE_PATH")
shoreline_feat = run['shoreline_feature']
# Set up output directory/bucket for run
output_formats = ['Shapefile', 'NetCDF', 'Trackline']
# Setup Models
models = []
if run['cached_behavior'] is not None and run['cached_behavior'].get('results', None) is not None:
behavior_data = run['cached_behavior']['results'][0]
l = LarvaBehavior(data=behavior_data)
models.append(l)
models.append(Transport(horizDisp=run['horiz_dispersion'], vertDisp=run['vert_dispersion']))
# Setup ModelController
model = ModelController(geometry=geometry, depth=start_depth, start=start_time, step=time_step, nstep=num_steps, npart=num_particles, models=models, use_bathymetry=True, use_shoreline=True,
time_chunk=run['time_chunk'], horiz_chunk=run['horiz_chunk'], time_method=run['time_method'], shoreline_path=shoreline_path, shoreline_feature=shoreline_feat, reverse_distance=1500)
# Run the model
cache_file = os.path.join(cache_path, run_id + ".nc.cache")
bathy_file = current_app.config['BATHY_PATH']
model.run(run['hydro_path'], output_path=output_path, bathy=bathy_file, output_formats=output_formats, cache=cache_file, remove_cache=False, caching=run['caching'])
# Skip creating movie output_path
"""
from paegan.viz.trajectory import CFTrajectory
logger.info("Creating animation...")
for filename in os.listdir(output_path):
if os.path.splitext(filename)[1][1:] == "nc":
# Found netCDF file
netcdf_file = os.path.join(output_path,filename)
traj = CFTrajectory(netcdf_file)
success = traj.plot_animate(os.path.join(output_path,'animation.avi'), temp_folder=temp_animation_path, bathy=app.config['BATHY_PATH'])
if not success:
logger.info("Could not create animation")
else:
logger.info("Animation saved")
"""
job.meta["outcome"] = "success"
job.save()
return "Successfully ran %s" % run_id
except Exception as exception:
logger.warn("Run FAILED, cleaning up and uploading log.")
logger.warn(exception.message)
job.meta["outcome"] = "failed"
job.save()
raise
finally:
logger.progress((99, "Processing output files"))
# Close the handler so we can upload the log file without a file lock
(hand.close() for hand in logger.handlers)
queue.put(StopIteration)
# Break out of the progress loop
e.set()
t.join()
# Move logfile to output directory
shutil.move(log_file, os.path.join(output_path, 'model.log'))
# Move cachefile to output directory if we made one
if run['caching']:
shutil.move(cache_file, output_path)
output_files = []
for filename in os.listdir(output_path):
outfile = os.path.join(output_path, filename)
output_files.append(outfile)
result_files = []
base_access_url = current_app.config.get('NON_S3_OUTPUT_URL', None)
# Handle results and cleanup
if current_app.config['USE_S3'] is True:
base_access_url = urljoin("http://%s.s3.amazonaws.com/output/" % current_app.config['S3_BUCKET'], run_id)
# Upload results to S3 and remove the local copies
conn = S3Connection()
bucket = conn.get_bucket(current_app.config['S3_BUCKET'])
for outfile in output_files:
# Don't upload the cache file
if os.path.basename(outfile) == os.path.basename(cache_file):
continue
# Upload the outfile with the same as the run name
_, ext = os.path.splitext(outfile)
new_filename = slugify(unicode(run['name'])) + ext
k = Key(bucket)
k.key = "output/%s/%s" % (run_id, new_filename)
k.set_contents_from_filename(outfile)
k.set_acl('public-read')
result_files.append(base_access_url + "/" + new_filename)
os.remove(outfile)
shutil.rmtree(output_path, ignore_errors=True)
else:
for outfile in output_files:
result_files.append(urljoin(base_access_url, run_id) + "/" + os.path.basename(outfile))
shutil.rmtree(temp_animation_path, ignore_errors=True)
# Set output fields
run.output = result_files
run.ended = datetime.utcnow()
run.compute()
run.save()
# Cleanup
logger.removeHandler(handler)
del formatter
del handler
del logger
del model
queue.close()
job.meta["message"] = "Complete"
job.save()
def manager(run_id):
with app.app_context():
job = get_current_job()
output_path = os.path.join(current_app.config['OUTPUT_PATH'], run_id)
shutil.rmtree(output_path, ignore_errors=True)
os.makedirs(output_path)
cache_path = os.path.join(current_app.config['CACHE_PATH'], run_id)
shutil.rmtree(cache_path, ignore_errors=True)
os.makedirs(cache_path)
f, log_file = tempfile.mkstemp(dir=cache_path, prefix=run_id, suffix=".log")
os.close(f)
os.chmod(log_file, 0644)
# Set up Logger
logger = logging.getLogger(run_id)
handler = FileHandler(log_file)
handler.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s] - %(levelname)s - %(name)s - %(processName)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
res = urlparse(current_app.config.get("RESULTS_REDIS_URI"))
redis_pool = redis.ConnectionPool(host=res.hostname, port=res.port, db=res.path[1:])
r = redis.Redis(connection_pool=redis_pool)
run = db.Run.find_one( { '_id' : ObjectId(run_id) } )
if run is None:
return "Failed to locate run %s. May have been deleted while task was in the queue?" % run_id
def listen_for_logs():
pubsub = r.pubsub()
pubsub.subscribe("%s:log" % run_id)
for msg in pubsub.listen():
if msg['type'] != "message":
continue
if msg["data"] == "FINISHED":
break
try:
prog = json.loads(msg["data"])
if prog is not None:
if prog.get("level", "").lower() == "progress":
job.meta["progress"] = float(prog.get("value", job.meta.get("progress", None)))
job.meta["message"] = prog.get("message", job.meta.get("message", ""))
job.meta["updated"] = prog.get("time", datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(pytz.utc))
job.save()
logger.info("PROGRESS: %(value).2f - %(message)s" % prog)
else:
getattr(logger, prog["level"].lower())(prog.get("message"))
except Exception:
logger.info("Got strange result: %s" % msg["data"])
pass
pubsub.close()
sys.exit()
def listen_for_results(output_h5_file, total_particles):
# Create output file (hdf5)
particles_finished = 0
results = ResultsPyTable(output_h5_file)
pubsub = r.pubsub()
pubsub.subscribe("%s:results" % run_id)
for msg in pubsub.listen():
if msg['type'] != "message":
continue
if msg["data"] == "FINISHED":
break
try:
json_msg = json.loads(msg["data"])
if json_msg.get("status", None):
# "COMPLETED" or "FAILED" when a particle finishes
particles_finished += 1
percent_complete = 90. * (float(particles_finished) / float(total_particles)) + 5 # Add the 5 progress that was used prior to the particles starting (controller)
r.publish("%s:log" % run_id, json.dumps({"time" : datetime.utcnow().isoformat(), "level" : "progress", "value" : percent_complete, "message" : "Particle #%s %s!" % (particles_finished, json_msg.get("status"))}))
if particles_finished == total_particles:
break
else:
# Write to HDF file
results.write(json_msg)
except Exception:
logger.info("Got strange result: %s" % msg["data"])
pass
pubsub.close()
results.compute()
results.close()
sys.exit()
pl = threading.Thread(name="LogListener", target=listen_for_logs)
pl.daemon = True
pl.start()
output_h5_file = os.path.join(output_path, "results.h5")
rl = threading.Thread(name="ResultListener", target=listen_for_results, args=(output_h5_file, run['particles']))
rl.daemon = True
rl.start()
# Wait for PubSub listening to begin
time.sleep(1)
model = None
try:
r.publish("%s:log" % run_id, json.dumps({"time" : datetime.utcnow().isoformat(), "level" : "progress", "value" : 0, "message" : "Setting up model"}))
hydropath = run['hydro_path']
geometry = loads(run['geometry'])
start_depth = run['release_depth']
num_particles = run['particles']
time_step = run['timestep']
num_steps = int(math.ceil((run['duration'] * 24 * 60 * 60) / time_step))
start_time = run['start'].replace(tzinfo = pytz.utc)
shoreline_path = run['shoreline_path'] or app.config.get("SHORE_PATH")
shoreline_feat = run['shoreline_feature']
# Setup Models
models = []
if run['cached_behavior'] is not None and run['cached_behavior'].get('results', None) is not None:
behavior_data = run['cached_behavior']['results'][0]
l = LarvaBehavior(data=behavior_data)
models.append(l)
models.append(Transport(horizDisp=run['horiz_dispersion'], vertDisp=run['vert_dispersion']))
model = DistributedModelController(geometry=geometry,
depth=start_depth,
start=start_time,
step=time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
bathy_path=current_app.config['BATHY_PATH'],
use_shoreline=True,
time_method=run['time_method'],
shoreline_path=shoreline_path,
shoreline_feature=shoreline_feat,
shoreline_index_buffer=0.05)
model.setup_run(hydropath, output_formats=["redis"], redis_url=current_app.config.get("RESULTS_REDIS_URI"), redis_results_channel="%s:results" % run_id, redis_log_channel="%s:log" % run_id)
except Exception as exception:
logger.warn("Run failed to initialize, cleaning up.")
logger.warn(exception.message)
job.meta["outcome"] = "failed"
job.save()
raise
try:
r.publish("%s:log" % run_id, json.dumps({"time" : datetime.utcnow().isoformat(), "level" : "progress", "value" : 4, "message" : "Adding particles to queue"}))
for part in model.particles:
particle_queue.enqueue_call(func=particle, args=(hydropath, part, model,))
except Exception, exception:
logger.warn("Failed to start particles, cleaning up.")
logger.warn(exception.message)
r.publish("%s:results" % run_id, "FINISHED")
job.meta["outcome"] = "failed"
job.save()
raise
finally:
def run(run_id):
with app.app_context():
job = get_current_job(connection=redis_connection)
output_path = os.path.join(current_app.config['OUTPUT_PATH'], run_id)
shutil.rmtree(output_path, ignore_errors=True)
os.makedirs(output_path)
cache_path = os.path.join(current_app.config['CACHE_PATH'], run_id)
shutil.rmtree(cache_path, ignore_errors=True)
os.makedirs(cache_path)
f, log_file = tempfile.mkstemp(dir=cache_path, prefix=run_id, suffix=".log")
os.close(f)
os.chmod(log_file, 0644)
run = db.Run.find_one( { '_id' : ObjectId(run_id) } )
if run is None:
return "Failed to locate run %s. May have been deleted while task was in the queue?" % run_id
# Wait for PubSub listening to begin
time.sleep(2)
model = None
try:
hydropath = run['hydro_path']
geometry = loads(run['geometry'])
start_depth = run['release_depth']
num_particles = run['particles']
time_step = run['timestep']
num_steps = int(math.ceil((run['duration'] * 24 * 60 * 60) / time_step))
start_time = run['start'].replace(tzinfo = pytz.utc)
shoreline_path = run['shoreline_path'] or app.config.get("SHORE_PATH")
shoreline_feat = run['shoreline_feature']
# Setup Models
models = []
if run['cached_behavior'] is not None and run['cached_behavior'].get('results', None) is not None:
behavior_data = run['cached_behavior']['results'][0]
l = LarvaBehavior(data=behavior_data)
models.append(l)
models.append(Transport(horizDisp=run['horiz_dispersion'], vertDisp=run['vert_dispersion']))
output_formats = [
ex.H5Trackline,
ex.H5TracklineWithPoints,
ex.H5ParticleTracklines,
ex.H5ParticleMultiPoint,
ex.H5GDALShapefile
]
def listen_for_logs(redis_url, log_channel, stop):
res = urlparse(redis_url)
redis_pool = redis.ConnectionPool(host=res.hostname, port=res.port, db=res.path[1:])
r = redis.Redis(connection_pool=redis_pool)
# Set up Logger
logger = logging.getLogger(run_id)
logger.setLevel(logging.PROGRESS)
handler = FileHandler(log_file)
handler.setLevel(logging.PROGRESS)
formatter = logging.Formatter('[%(asctime)s] - %(levelname)s - %(name)s - %(processName)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
if app.config.get("DEBUG") is True:
st = logging.StreamHandler()
st.setLevel(logging.PROGRESS)
st.setFormatter(formatter)
logger.addHandler(st)
pubsub = r.pubsub()
pubsub.subscribe(log_channel)
while True:
if stop():
break
msg = pubsub.get_message()
if msg:
if msg['type'] != "message":
continue
try:
prog = json.loads(msg["data"])
if prog is not None:
if prog.get("level", "").lower() == "progress":
job.meta["progress"] = float(prog.get("value", job.meta.get("progress", None)))
job.meta["message"] = prog.get("message", job.meta.get("message", ""))
job.meta["updated"] = prog.get("time", datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(pytz.utc))
job.save()
logger.progress("%(value).2f - %(message)s" % prog)
else:
getattr(logger, prog["level"].lower())(prog.get("message"))
except Exception:
logger.info("Got strange result: %s" % msg["data"])
# Relax
time.sleep(0.01)
pubsub.unsubscribe()
pubsub.close()
# Close and remove the handlers so we can use the log file without a file lock
for hand in list(logger.handlers):
logger.removeHandler(hand)
hand.flush()
hand.close()
del hand
# Remove all connections
redis_pool.disconnect()
sys.exit()
stop_log_listener = False
log_channel = "{}:log".format(run_id)
results_channel = "{}:results".format(run_id)
pl = threading.Thread(name="LogListener", target=listen_for_logs, args=(current_app.config.get("RESULTS_REDIS_URI"), log_channel, lambda: stop_log_listener, ))
pl.daemon = True
pl.start()
model = DistributedModelController(
geometry=geometry,
depth=start_depth,
start=start_time,
step=time_step,
nstep=num_steps,
npart=num_particles,
models=models,
use_bathymetry=True,
bathy_path=current_app.config['BATHY_PATH'],
use_shoreline=True,
time_method=run['time_method'],
shoreline_path=shoreline_path,
shoreline_feature=shoreline_feat,
shoreline_index_buffer=0.05)
model.setup_run(hydropath, redis_url=current_app.config.get("RESULTS_REDIS_URI"), redis_results_channel=results_channel, redis_log_channel=log_channel)
run.started = datetime.utcnow()
model.run(output_formats=output_formats, output_path=output_path, task_queue_call=particle_queue.enqueue_call)
except Exception as e:
# Send message to the log listener to finish
stop_log_listener = True
# Wait for log listener to exit
pl.join()
app.logger.exception("Failed model run")
job.meta["message"] = e.message
job.meta["outcome"] = "failed"
else:
# Send message to the log listener to finish
stop_log_listener = True
# Wait for log listener to exit
pl.join()
job.meta["message"] = "Complete"
job.meta["outcome"] = "success"
job.meta["progress"] = 100
finally:
# Move logfile to output directory
time.sleep(1)
shutil.move(log_file, os.path.join(output_path, 'model.log'))
output_files = []
for filename in os.listdir(output_path):
outfile = os.path.join(output_path, filename)
output_files.append(outfile)
result_files = []
# Handle results and cleanup
if current_app.config['USE_S3'] is True:
base_access_url = urljoin("http://%s.s3.amazonaws.com/output/" % current_app.config['S3_BUCKET'], run_id)
# Upload results to S3 and remove the local copies
conn = S3Connection()
bucket = conn.get_bucket(current_app.config['S3_BUCKET'])
for outfile in output_files:
# Upload the outfile with the same as the run name
_, ext = os.path.splitext(outfile)
new_filename = slugify(unicode(run['name'])) + ext
k = Key(bucket)
k.key = "output/%s/%s" % (run_id, new_filename)
k.set_contents_from_filename(outfile)
k.set_acl('public-read')
result_files.append(base_access_url + "/" + new_filename)
os.remove(outfile)
shutil.rmtree(output_path, ignore_errors=True)
else:
result_files = output_files
job.meta["updated"] = datetime.utcnow().replace(tzinfo=pytz.utc).astimezone(pytz.utc)
job.save()
# Set output fields
run.output = result_files
run.ended = datetime.utcnow()
run.compute()
run.save()
def shoreline_choices():
with app.app_context():
return [(s.path, s.name) for s in db.Shoreline.find().sort('name')]
def dataset_choices():
with app.app_context():
return [(s.location, '{}: {:%b %d %Y} to {:%b %d %Y}'.format(s.name, s.starting, s.ending) ) for s in db.Dataset.find().sort('name')]
