def do_pipeline():
# Cleaning up old files from the database and the disk
DataFileManager.delete_old_files()
OverlayManager.delete_old_files()
# Check for new feedback surveys or comments, and email them to Flaxen
FeedbackHistory.send_feedback_forms()
FeedbackQuestionaire.send_feedback_survey()
#Downloading the latest datafiles for our models. See the appropriate functions
#pl_download/models.py.DataFileManager.get_latest_wave_watch_files() and
#pl_download/models.py.DataFileManager.fetch_new_files() respectively
try:
#Sometimes even though the file downloads this process hangs and fails.
#The try catch is a stop-gap fix so that the pipeline doesn't stop here
#When it fails in that manner the file is downloaded and can be used
wave_watch_files = DataFileManager.get_latest_wave_watch_files()
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
sst_files = DataFileManager.fetch_new_files() # not calling as a task so it runs inline
wind_files = DataFileManager.get_wind_file()
# If no new files were returned, don't plot or tile anything.
try:
#This try catch is also for the wave watch timeout bug
if not wave_watch_files and not sst_files and not wind_files:
print "No New Files Available, Quitting."
return None
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
# get the list of plotting tasks based on the files we just downloaded.
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days()
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4 and pt.args[0] != 6 and pt.args[0] != 7:
# chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
#Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
job = group(item for item in list_of_chains)
print "jobs:"
for each in job:
print each
#and run the group.
result = job.apply_async()
return result
def home(request):
#Models determines which models are displayed on the website. They will appear in the order provided by models[]. Change this order to change the order of the buttons and which buttons appear.
# OSU_ROMS_SST = 1
# OSU_ROMS_SUR_SAL = 2
# OSU_ROMS_SUR_CUR = 3
# OSU_WW3_HI = 4
# NAMS_WIND = 5
# OSU_WW3_DIR = 6
# OSU_ROMS_BOT_SAL = 7
# OSU_ROMS_BOT_TEMP = 8
# OSU_ROMS_SSH = 9
# NCEP_WW3_DIR = 10
# NCEP_WW3_HI = 11
# HYCOM_SST = 12 # RTOFS - Not Used - needs to be renamed
# HYCOM_SUR_CUR = 13 # RTOFS - Not Used - needs to be renamed
# OSU_ROMS_TCLINE = 14
# OSU_ROMS_PCLINE = 15
# NAVY_HYCOM_SST = 16
# NAVY_HYCOM_SUR_CUR = 17
# NAVY_HYCOM_SUR_SAL = 18
# NAVY_HYCOM_SSH = 19
# NAVY_HYCOM_BOT_TEMP = 20
# NAVY_HYCOM_BOT_CUR = 21
# NAVY_HYCOM_BOT_SAL = 22
models = [settings.OSU_ROMS_SST,
settings.OSU_ROMS_SUR_CUR,
settings.NCEP_WW3_HI,
settings.NCEP_WW3_DIR,
settings.NAMS_WIND,
settings.OSU_ROMS_BOT_TEMP,
settings.OSU_ROMS_SUR_SAL,
settings.OSU_ROMS_BOT_SAL,
settings.OSU_ROMS_SSH,
settings.OSU_ROMS_TCLINE,
]
extended_models = [settings.OSU_ROMS_SST,
settings.OSU_ROMS_SUR_CUR]
fields = get_list_of_overlay_definitions(models)
base_overlays = OverlayManager.get_next_few_days_of_tiled_overlays(models)
if settings.EXTEND:
extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts('NCDF')
overlays = base_overlays | extended_overlays # Union of all three querysets - '|' represents the union
else:
overlays = base_overlays
datetimes = overlays.values_list('applies_at_datetime', flat=True).distinct().order_by('applies_at_datetime')
context = {'overlays': overlays, 'defs': fields, 'times':datetimes }
return render(request, 'index.html', context)
def test(ids=None, navy=False, ncep=False):
from pl_download.models import DataFile
from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
from pl_plot.models import OverlayManager as om
print ids
if navy:
print "NAVY HYCOM TEST"
count = 0
if ids:
for id in ids:
datafile = DataFile.objects.get(pk=id)
if datafile.type != 'HYCOM':
continue # Skip over non HYCOM Files
count += 1
print "HYCOM TESTING ", count, " FOR FILE: "
info(datafile)
print "\tTESTING PLOTTER:"
plotter = NavyPlotter(datafile.file.name)
if plotter:
print "\t PLOTTER LOADED SUCCESFULLY"
else:
print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name
print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()
print "\t GENERATING PLOT.... "
print ""
if datafile.file.name.endswith("ssh.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("temp_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
if datafile.file.name.endswith("temp_bot.nc"):
om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
if datafile.file.name.endswith("cur_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("sal_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)
if ncep:
from pl_download.models import DataFileManager as dm
print dm.ww3_download_openDAP()
print "TESTING TASK CREATION"
def test(ids=None, navy=False, ncep=False):
from pl_download.models import DataFile
from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
from pl_plot.models import OverlayManager as om
print ids
if navy:
print "NAVY HYCOM TEST"
count = 0
if ids:
for id in ids:
datafile = DataFile.objects.get(pk=id)
if datafile.type != 'HYCOM':
continue # Skip over non HYCOM Files
count += 1
print "HYCOM TESTING ", count, " FOR FILE: "
info(datafile)
print "\tTESTING PLOTTER:"
plotter = NavyPlotter(datafile.file.name)
if plotter:
print "\t PLOTTER LOADED SUCCESFULLY"
else:
print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name
print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times(
)
print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()
print "\t GENERATING PLOT.... "
print ""
if datafile.file.name.endswith("ssh.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("temp_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
if datafile.file.name.endswith("temp_bot.nc"):
om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
if datafile.file.name.endswith("cur_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("sal_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)
if ncep:
from pl_download.models import DataFileManager as dm
print dm.ww3_download_openDAP()
print "TESTING TASK CREATION"
def home(request):
#overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays().exclude(definition_id=5)
overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays()
#Wind overlay is different from the main overlay due to the time intervals
#wind_overlays_view_data = overlays_view_data.filter(definition_id=5)
datetimes = [ i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p') for i in overlays_view_data ]
#winddatetimes = [ i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p') for i in wind_overlays_view_data ]
# a complete hack! it just divides a list of all of the times for all the overlays by the number
# of defs to get a singular list of overlay times
# num_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="Wind"))
# num_wind_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="SST").exclude(display_name_short="Currents"))
# Team 2 says: at this time we want SST (1), currents (3), and Wave (4) only. Modify this as
# you add new models.
num_defs = len(OverlayDefinition.objects.filter(is_base=True, id__in=[1,3,4]))
list_of_times = datetimes[:len(datetimes)/num_defs]
#list_of_wind_times = winddatetimes[:len(winddatetimes)/num_wind_defs]
#context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True).exclude(id=4), 'times':list_of_times, 'windoverlays': wind_overlays_view_data, 'winddefs': OverlayDefinition.objects.filter(id=5), 'windtimes':list_of_wind_times}
context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True, id__in=[1,3,4]), 'times':list_of_times }
return render(request, 'index.html', context)
def testplot(request):
#Validate User Permissions
if request.user.is_anonymous():
return HttpResponse("You do not have admin permissions to this site. Please contact your page administrator.")
else:
group_results = OverlayManager.make_all_base_plots_for_next_few_days()
name_list = [result.get() for result in group_results]
return HttpResponse("plotted files " + name_list.__str__())
def extend():
from pl_plot.models import OverlayManager
models = [
settings.OSU_ROMS_SST, # Extended
settings.OSU_ROMS_SUR_SAL,
settings.OSU_ROMS_SUR_CUR, # Extended
settings.OSU_WW3_HI, # Extended
settings.NAMS_WIND,
settings.OSU_WW3_DIR, # Extended
settings.OSU_ROMS_BOT_SAL,
settings.OSU_ROMS_BOT_TEMP,
settings.OSU_ROMS_SSH,
settings.OSU_ROMS_TCLINE,
]
extended_models = [
settings.OSU_ROMS_SST, settings.OSU_ROMS_SUR_CUR, settings.OSU_WW3_HI,
settings.OSU_WW3_DIR
]
# 14 = Thermocline
#models = [1,3,4,6,5,8,2,7,9,]
fields = get_list_of_overlay_definitions(models)
base_overlays = OverlayManager.get_next_few_days_of_tiled_overlays(models)
ww3_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts(
'WAVE', extended_models)
roms_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts(
'NCDF', extended_models)
print "Base Overlays:"
for overlay in base_overlays:
print overlay.file, overlay.applies_at_datetime, overlay.is_extend
print "Extended WW3"
for overlay in ww3_extended_overlays:
print overlay.file, overlay.applies_at_datetime, overlay.is_extend
print "Extended ROMS"
for overlay in roms_extended_overlays:
print overlay.file, overlay.applies_at_datetime, overlay.is_extend
overlays = base_overlays | ww3_extended_overlays | roms_extended_overlays # Union of all three querysets - '|' represents the union
def home(request):
#This determines which models are ran
#1 = SST, 3 = Currents, 4 = Wave Height, 5 = Winds, 6 = Wave Direction
#-------------------------------------------------------------------------
models = [1,3,4,5,6]
overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays(models)
datetimes = overlays_view_data.values_list('applies_at_datetime', flat=True).distinct().order_by('applies_at_datetime')
context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True, id__in=models), 'times':datetimes }
return render(request, 'index.html', context)
def testplot(request):
#Validate User Permissions
if request.user.is_anonymous():
return HttpResponse(
"You do not have admin permissions to this site. Please contact your page administrator."
)
else:
group_results = OverlayManager.make_all_base_plots_for_next_few_days()
name_list = [result.get() for result in group_results]
return HttpResponse("plotted files " + name_list.__str__())
def extend():
from pl_plot.models import OverlayManager
models = [settings.OSU_ROMS_SST, # Extended
settings.OSU_ROMS_SUR_SAL,
settings.OSU_ROMS_SUR_CUR, # Extended
settings.OSU_WW3_HI, # Extended
settings.NAMS_WIND,
settings.OSU_WW3_DIR, # Extended
settings.OSU_ROMS_BOT_SAL,
settings.OSU_ROMS_BOT_TEMP,
settings.OSU_ROMS_SSH,
settings.OSU_ROMS_TCLINE,
]
extended_models = [settings.OSU_ROMS_SST,
settings.OSU_ROMS_SUR_CUR,
settings.OSU_WW3_HI,
settings.OSU_WW3_DIR]
# 14 = Thermocline
#models = [1,3,4,6,5,8,2,7,9,]
fields = get_list_of_overlay_definitions(models)
base_overlays = OverlayManager.get_next_few_days_of_tiled_overlays(models)
ww3_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts('WAVE', extended_models)
roms_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts('NCDF', extended_models)
print "Base Overlays:"
for overlay in base_overlays:
print overlay.file, overlay.applies_at_datetime, overlay.is_extend
print "Extended WW3"
for overlay in ww3_extended_overlays:
print overlay.file, overlay.applies_at_datetime, overlay.is_extend
print "Extended ROMS"
for overlay in roms_extended_overlays:
print overlay.file, overlay.applies_at_datetime, overlay.is_extend
overlays = base_overlays | ww3_extended_overlays | roms_extended_overlays # Union of all three querysets - '|' represents the union
def do_pipeline():
print "TASKS: before deleting files"
DataFileManager.delete_old_files()
print "TASKS: after deleting Datafiles"
OverlayManager.delete_old_files()
print "TASKS: after deleting Overlays"
wave_watch_files = DataFileManager.get_latest_wave_watch_files()
print "TASKS: after getting wave files"
other_files = DataFileManager.fetch_new_files(
) # not calling as a task so it runs inline
print "TASKS: after getting sst/currents files"
# If no new files were returned, don't plot or tile anything.
if not wave_watch_files and not other_files:
return None
# get the list of plotting tasks based on the files we just downloaded.
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days(
)
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4:
# chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
#Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
job = group(item for item in list_of_chains)
print "jobs:"
for each in job:
print each
#and run the group.
result = job.apply_async()
return result
def home(request):
#overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays().exclude(definition_id=5)
overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays()
#Wind overlay is different from the main overlay due to the time intervals
#wind_overlays_view_data = overlays_view_data.filter(definition_id=5)
datetimes = [
i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p')
for i in overlays_view_data
]
#winddatetimes = [ i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p') for i in wind_overlays_view_data ]
# a complete hack! it just divides a list of all of the times for all the overlays by the number
# of defs to get a singular list of overlay times
# num_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="Wind"))
# num_wind_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="SST").exclude(display_name_short="Currents"))
# Team 2 says: at this time we want SST (1), currents (3), and Wave (4) only. Modify this as
# you add new models.
num_defs = len(
OverlayDefinition.objects.filter(is_base=True, id__in=[1, 3, 4]))
list_of_times = datetimes[:len(datetimes) / num_defs]
#list_of_wind_times = winddatetimes[:len(winddatetimes)/num_wind_defs]
#context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True).exclude(id=4), 'times':list_of_times, 'windoverlays': wind_overlays_view_data, 'winddefs': OverlayDefinition.objects.filter(id=5), 'windtimes':list_of_wind_times}
context = {
'overlays': overlays_view_data,
'defs': OverlayDefinition.objects.filter(is_base=True,
id__in=[1, 3, 4]),
'times': list_of_times
}
return render(request, 'index.html', context)
def plot(ids=[],
num_plots=DEF_NUM_PLOTS,
tile_flag=DEF_TILE_FLAG,
full_roms=DEF_FULL_ROMS_FLAG,
roms=False,
wave=False,
wind=False,
hycom=False,
ncep=False,
tcline=False,
navy=False):
''' Just generates plots. You need to pass in the df id to get a plot! Pass it in manually
or by using one of the functions below which grabs them using the database or via downloading!
'''
print "IDS", ids
if not ids:
print "PLOT: NO IDS SUBMITTED TO BE PLOTTED - exiting"
return
if len(ids) == 0:
print "PLOT: Empty List of IDS exiting"
return
from pl_plot.models import OverlayManager as om
from pl_chop.tasks import tile_overlay
if roms: # OSU ROMS
roms = []
print "PLOT: Plotting Roms with file IDS: ", ids
for id in ids:
for i in range(num_plots):
print "PLOT: OSU ROMS SST - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SST, i, id))
print "PLOT: OSU ROMS SSC - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SUR_CUR, i, id))
if full_roms:
print "PLOT: Plotting full roms"
print "PLOT: OSU ROMS SSC - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SUR_SAL, i, id))
print "PLOT: OSU ROMS BOT Sal- timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_BOT_SAL, i, id))
print "PLOT: OSU ROMS BOT Temp- timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_BOT_TEMP, i,
id))
print "PLOT: OSU ROMS SSH - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SSH, i, id))
if tile_flag:
print "PLOT: Tiling ROMS"
tile(roms)
return
if wave: # OSU WAVE WATCH III - NO LONGER AVIABLE
waves = []
print ids
print "PLOT: Plotting OSU WW3 with file IDS: ", ids
if not ids:
print "Empty List of IDS for OSU WW3"
return
for id in ids:
for i in range(num_plots):
waves.append(
om.make_wave_watch_plot(settings.OSU_WW3_HI, i, id))
waves.append(
om.make_wave_watch_plot(settings.OSU_WW3_DIR, i, id))
if tile_flag:
print "PLOT: Tiling waves"
tile(waves)
return
if wind: # NORTH AMERICAN MESOSCALE - SURFACE WINDS
winds = []
print "PLOT: Plotting NAM Winds with file IDS: ", ids
for id in ids:
for i in range(num_plots):
winds.append(om.make_plot(settings.NAMS_WIND, i, id))
if tile_flag:
print "PLOT: Tiling NAM Winds"
tile(winds)
return
if hycom: # NOAA HYCOM - Not currently implemented on the seacast.org
hycoms = []
print "PLOT: Plotting HYCOM with file IDS: ", ids
for id in ids:
hycoms.append(om.make_plot(settings.HYCOM_SST, 0, id))
hycoms.append(om.make_plot(settings.HYCOM_SUR_CUR, 0, id))
if tile_flag:
print "PLOT: Tiling HYCOM"
tile(hycoms)
return
if ncep: # NCEP WAVE WATCH III
nceps = []
print ids
print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
for id in ids:
for i in range(num_plots):
nceps.append(
om.make_wave_watch_plot(settings.NCEP_WW3_DIR, i, id))
nceps.append(
om.make_wave_watch_plot(settings.NCEP_WW3_HI, i, id))
if tile_flag:
print "PLOT: Tiling NCEP"
tile(nceps)
return
if tcline: # OSU ROMS THERMOCLINE
tcline_ids = []
print ids
print "PLOT: Plotting TCLINE with file IDS: ", ids
for id in ids:
for i in range(num_plots):
tcline_ids.append(om.make_plot(settings.OSU_ROMS_TCLINE, i,
id))
if tile_flag:
print "PLOT: Tiling tcline"
tile(tcline_ids)
return
if navy: # NAVY HYCOM
navy_ids = []
print ids
print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
for id in ids:
for i in range(num_plots):
navy_ids.append(om.make_plot(settings.NAVY_HYCOM_SST, i, id))
navy_ids.append(
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, i, id))
if tile_flag:
print "PLOT: Tiling NCEP"
tile(navy_ids)
return
def get_task_to_tile_next_few_days_of_untiled_overlays(cls):
overlay_ids_to_tile = OverlayManager.get_next_few_days_of_untiled_overlay_ids(
)
return cls.get_tiling_task_by_ids(overlay_ids_to_tile)
if sys.argv[-1] == "plot":
from pl_download.models import DataFileManager, DataFile
from pl_plot.models import OverlayManager
from pl_chop.tasks import tile_overlay, tile_wave_watch_overlay
wave = 0
sst = 0
wind = 1
if wave:
wave = DataFileManager.get_latest_wave_watch_files()
wave = DataFile.objects.filter(type='WAVE').latest('model_date')
tiles = []
begin = time.time()
#first entry is day-1 at 12pm
#need to offset 16 to match with sst plot
#NOTE it increments in 1 hour changes
tiles += OverlayManager.make_wave_watch_plot(4, 16, wave[0])
tiles += OverlayManager.make_wave_watch_plot(6, 16, wave[0])
for t in tiles:
tile_wave_watch_overlay(t)
finish = time.time()
totalTime = (finish - begin)/ 60
print "Time taken for Waves = " + str(round(totalTime, 2)) + " minutes"
if sst:
sst = DataFileManager.fetch_new_files()
tiles = []
#first entry is day at 4am
#NOTE it increments in 4 hour changes
begin = time.time()
tiles += OverlayManager.make_plot(1, 0, sst[0])
tiles += OverlayManager.make_plot(3, 0, sst[0])
for t in tiles:
def plot(ids=[],
num_plots=DEF_NUM_PLOTS, tile_flag=DEF_TILE_FLAG, full_roms=DEF_FULL_ROMS_FLAG,
roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False):
''' Just generates plots. You need to pass in the df id to get a plot! Pass it in manually
or by using one of the functions below which grabs them using the database or via downloading!
'''
print "IDS", ids
if not ids:
print "PLOT: NO IDS SUBMITTED TO BE PLOTTED - exiting"
return
if len(ids) == 0:
print "PLOT: Empty List of IDS exiting"
return
from pl_plot.models import OverlayManager as om
from pl_chop.tasks import tile_overlay
if roms: # OSU ROMS
roms = []
print "PLOT: Plotting Roms with file IDS: ", ids
for id in ids:
for i in range(num_plots):
print "PLOT: OSU ROMS SST - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SST, i, id))
print "PLOT: OSU ROMS SSC - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SUR_CUR, i, id))
if full_roms:
print "PLOT: Plotting full roms"
print "PLOT: OSU ROMS SSC - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SUR_SAL, i, id))
print "PLOT: OSU ROMS BOT Sal- timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_BOT_SAL, i, id))
print "PLOT: OSU ROMS BOT Temp- timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_BOT_TEMP, i, id))
print "PLOT: OSU ROMS SSH - timeslice: ", i
roms.append(om.make_plot(settings.OSU_ROMS_SSH, i, id))
if tile_flag:
print "PLOT: Tiling ROMS"
tile(roms)
return
if wave: # OSU WAVE WATCH III - NO LONGER AVIABLE
waves = []
print ids
print "PLOT: Plotting OSU WW3 with file IDS: ", ids
if not ids:
print "Empty List of IDS for OSU WW3"
return
for id in ids:
for i in range(num_plots):
waves.append(om.make_wave_watch_plot(settings.OSU_WW3_HI, i, id))
waves.append(om.make_wave_watch_plot(settings.OSU_WW3_DIR, i, id))
if tile_flag:
print "PLOT: Tiling waves"
tile(waves)
return
if wind: # NORTH AMERICAN MESOSCALE - SURFACE WINDS
winds = []
print "PLOT: Plotting NAM Winds with file IDS: ", ids
for id in ids:
for i in range(num_plots):
winds.append(om.make_plot(settings.NAMS_WIND, i, id))
if tile_flag:
print "PLOT: Tiling NAM Winds"
tile(winds)
return
if hycom: # NOAA HYCOM - Not currently implemented on the seacast.org
hycoms = []
print "PLOT: Plotting HYCOM with file IDS: ", ids
for id in ids:
hycoms.append(om.make_plot(settings.HYCOM_SST, 0, id))
hycoms.append(om.make_plot(settings.HYCOM_SUR_CUR, 0, id))
if tile_flag:
print "PLOT: Tiling HYCOM"
tile(hycoms)
return
if ncep: # NCEP WAVE WATCH III
nceps = []
print ids
print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
for id in ids:
for i in range(num_plots):
nceps.append(om.make_wave_watch_plot(settings.NCEP_WW3_DIR, i, id))
nceps.append(om.make_wave_watch_plot(settings.NCEP_WW3_HI, i, id))
if tile_flag:
print "PLOT: Tiling NCEP"
tile(nceps)
return
if tcline: # OSU ROMS THERMOCLINE
tcline_ids = []
print ids
print "PLOT: Plotting TCLINE with file IDS: ", ids
for id in ids:
for i in range(num_plots):
tcline_ids.append(om.make_plot(settings.OSU_ROMS_TCLINE, i, id))
if tile_flag:
print "PLOT: Tiling tcline"
tile(tcline_ids)
return
if navy: # NAVY HYCOM
navy_ids = []
print ids
print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
for id in ids:
for i in range(num_plots):
navy_ids.append(om.make_plot(settings.NAVY_HYCOM_SST, i, id))
navy_ids.append(om.make_plot(settings.NAVY_HYCOM_SUR_CUR, i, id))
if tile_flag:
print "PLOT: Tiling NCEP"
tile(navy_ids)
return
def test(ids=None, navy=False, ncep=False):
from pl_download.models import DataFile
from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
from pl_plot.models import OverlayManager as om
print ids
if navy:
print "NAVY HYCOM TEST"
count = 0
if ids:
for id in ids:
datafile = DataFile.objects.get(pk=id)
if datafile.type != 'HYCOM':
continue # Skip over non HYCOM Files
count += 1
print "HYCOM TESTING ", count, " FOR FILE: "
info(datafile)
print "\tTESTING PLOTTER:"
plotter = NavyPlotter(datafile.file.name)
if plotter:
print "\t PLOTTER LOADED SUCCESFULLY"
else:
print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name
print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()
print "\t GENERATING PLOT.... "
print ""
if datafile.file.name.endswith("ssh.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("temp_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
if datafile.file.name.endswith("temp_bot.nc"):
om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
if datafile.file.name.endswith("cur_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("sal_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)
if ncep:
print "NCEP TEST"
count = 0
for id in ids:
datafile = DataFile.objects.get(pk=id)
if datafile.type != 'NCEP':
continue # Skip over non HYCOM Files
count += 1
print "NCEP WW3 TESTING ", count, " FOR FILE: "
info(datafile)
print " TESTING PLOTTER:"
plotter = NcepWW3Plotter(datafile.file.name)
if plotter:
print " PLOTTER LOADED SUCCESFULLY"
else:
print " ERROR: UNABLE TO LOAD FILE: ", datafile.file.name
print "\tNUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
print "\tOCEAN TIMES & TASKS: "
list = om.get_tasks_for_base_plots_for_next_few_days()
for i in range(plotter.get_number_of_model_times()):
print "\t", list[i], "-- time:", plotter.get_oceantime(i)
print " GENERATING PLOTS.... "
cnt = 0
for i in range(plotter.get_number_of_model_times()):
print " PLOT: ", cnt, " of", plotter.get_number_of_model_times()
om.make_wave_watch_plot(settings.NCEP_WW3_DIR, i, id)
cnt += 1
print " PLOT: ", cnt, " of", plotter.get_number_of_model_times()
om.make_wave_watch_plot(settings.NCEP_WW3_HI, i, id)
cnt += 1
print "TESTING TASK CREATION"
def do_pipeline():
logging.info('DO_PIPELINE STARTED')
# Cleaning up old files from the database and the disk
print "CLEANING UP - DELETING OLD FILES"
logging.info('CLEANING UP - DELETING OLD DATAFILE FILES')
try:
DataFileManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OLD NETCDF FILES"
logging.error('CLEANING UP - ERROR DELETING DATAFILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD DATAFILE FILES DELETED')
logging.info('CLEANING UP - DELETING OLD OVERLAY FILES')
try:
OverlayManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OVERLAY FILES"
logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED')
# Check for new feedback surveys or comments, and email them to Flaxen
print "SENDING OUT FEEDBACK"
logging.info('SENDING OUT FEEDBACK')
try:
FeedbackHistory.send_feedback_forms()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK"
logging.error('ERROR SENDING FEEDBACK FORMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('FEEDBACK FORMS SENT')
logging.info('SENDING OUT SURVEY')
try:
FeedbackQuestionaire.send_feedback_survey()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK SURVEY"
logging.error('ERROR SENDING FEEDBACK SURVEY')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('SURVEYS Sent')
print "DOWNLOADING OSU ROMS FILES"
logging.info('DOWNLOADING OSU ROMS')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
sst_files = DataFileManager.download_osu_roms()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD OSU ROMS FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
if settings.WW3_OPENDAP is True:
print "DOWNLOADING NCEP FILES VIA OPENDAP"
logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download_openDAP()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP"
logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY')
else:
print "DOWNLOADING UCAR NCEP FILES"
logging.info('DOWNLOADING UCAR NCEP WW3')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES"
logging.error('ERROR DOWNLOADING UCAR NCEP WW3')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU WW3 DOWNLOADED SUCCESFULLY')
if settings.EXTEND:
print "DOWNLOADING NAVY HYCOM"
logging.info('DOWNLOADING NAVY HYCOM')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
hycom_files = DataFileManager.navy_hycom_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NAVY HYCOM FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING WIND FILES"
logging.info('DOWNLOADING NAM WIND')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wind_files = DataFileManager.get_wind_file()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD WIND FILE"
logging.error('ERROR DOWNLOADING NAM WINDS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NAM WINDS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING TCLINE FILES"
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
tcline_files = DataFileManager.download_tcline()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD TCLINE FILE"
traceback.print_exc(file=sys.stdout)
print '-' * 60
try:
#This try catch is also for the wave watch timeout bug
if not wave_watch_files and not sst_files and not wind_files \
and not hycom_files and not tcline_files:
print "No New Files Available, Quitting."
return None
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
# Get the list of plotting tasks based on the files we just downloaded.
logging.info('GENERATING TASK LIST')
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days()
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4 and pt.args[0] != 6:
# Chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
# Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
logging.info('TASK LIST GENERATED')
job = group(item for item in list_of_chains)
print "PIPELINE: JOBS: "
for each in job:
print each
logging.info('APPLY JOBS')
result = job.apply_async() # Run the group.
logging.info('JOBS APPLIED')
return result
def get_task_to_tile_next_few_days_of_untiled_overlays(cls):
overlay_ids_to_tile = OverlayManager.get_next_few_days_of_untiled_overlay_ids()
return cls.get_tiling_task_by_ids(overlay_ids_to_tile)
def do_pipeline():
logging.info('DO_PIPELINE STARTED')
# Cleaning up old files from the database and the disk
print "CLEANING UP - DELETING OLD FILES"
logging.info('CLEANING UP - DELETING OLD DATAFILE FILES')
try:
DataFileManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OLD NETCDF FILES"
logging.error('CLEANING UP - ERROR DELETING DATAFILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD DATAFILE FILES DELETED')
logging.info('CLEANING UP - DELETING OLD OVERLAY FILES')
try:
OverlayManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OVERLAY FILES"
logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED')
# Check for new feedback surveys or comments, and email them to Flaxen
print "SENDING OUT FEEDBACK"
logging.info('SENDING OUT FEEDBACK')
try:
FeedbackHistory.send_feedback_forms()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK"
logging.error('ERROR SENDING FEEDBACK FORMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('FEEDBACK FORMS SENT')
logging.info('SENDING OUT SURVEY')
try:
FeedbackQuestionaire.send_feedback_survey()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK SURVEY"
logging.error('ERROR SENDING FEEDBACK SURVEY')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('SURVEYS Sent')
print "DOWNLOADING OSU ROMS FILES"
logging.info('DOWNLOADING OSU ROMS')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
sst_files = DataFileManager.download_osu_roms()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD OSU ROMS FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
if settings.WW3_OPENDAP is True:
print "DOWNLOADING NCEP FILES VIA OPENDAP"
logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download_openDAP()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP"
logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY')
else:
print "DOWNLOADING UCAR NCEP FILES"
logging.info('DOWNLOADING UCAR NCEP WW3')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES"
logging.error('ERROR DOWNLOADING UCAR NCEP WW3')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU WW3 DOWNLOADED SUCCESFULLY')
if settings.EXTEND:
print "DOWNLOADING NAVY HYCOM"
logging.info('DOWNLOADING NAVY HYCOM')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
hycom_files = DataFileManager.navy_hycom_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NAVY HYCOM FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING WIND FILES"
logging.info('DOWNLOADING NAM WIND')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wind_files = DataFileManager.get_wind_file()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD WIND FILE"
logging.error('ERROR DOWNLOADING NAM WINDS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NAM WINDS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING TCLINE FILES"
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
tcline_files = DataFileManager.download_tcline()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD TCLINE FILE"
traceback.print_exc(file=sys.stdout)
print '-' * 60
try:
#This try catch is also for the wave watch timeout bug
if not wave_watch_files and not sst_files and not wind_files \
and not hycom_files and not tcline_files:
print "No New Files Available, Quitting."
return None
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
# Get the list of plotting tasks based on the files we just downloaded.
logging.info('GENERATING TASK LIST')
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days(
)
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4 and pt.args[0] != 6:
# Chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
# Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
logging.info('TASK LIST GENERATED')
job = group(item for item in list_of_chains)
print "PIPELINE: JOBS: "
for each in job:
print each
logging.info('APPLY JOBS')
result = job.apply_async() # Run the group.
logging.info('JOBS APPLIED')
return result
def plot_by_id(ids=None):
if ids == None:
print "PLOT_BY_ID: NO IDS SUPPLIED ... Quitting"
return 0
from pl_download.models import DataFile as df
from pl_plot.models import OverlayManager as om
for id in ids:
file = df.objects.get(pk=id)
print_file_info(file)
if file.type == 'NCDF':
om.make_plot()
elif file.type == 'WAVE':
om.make_plot()
elif file.type == 'WIND':
om.make_plot()
elif file.type == 'T-CLINE':
om.make_plot()
elif file.type == 'NCEP_WW3':
om.make_plot()
elif file.type == 'HYCOM':
om.make_plot()
elif file.type == 'RTOFS':
om.make_plot()
