def get_next_few_days_of_tiled_overlays_for_extended_forecasts(cls, type, is_tiled=True, is_extend=True):
"""
TODO: Update Docstring for this fucntion
:param type:
:param models:
:param is_tiled:
:param is_extend:
:return: TODO: Find out exactly what this function returns
"""
extend_date = None
if type == 'NCEP':
extend_date = DataFileManager.get_last_forecast_for_osu_ww3()
ids = [settings.OSU_WW3_HI, settings.OSU_WW3_DIR]
elif type == 'NCDF':
extend_date = DataFileManager.get_last_forecast_for_roms()
ids = [settings.OSU_ROMS_SST, settings.OSU_ROMS_SUR_CUR]
else:
print "Wrong type! Returning!"
return -1
dates = []
print "EXTEND DATE: ", extend_date
for id in ids: # Grab the dates based on the extend date found above
dates = Overlay.objects.filter(applies_at_datetime__gte=extend_date,
is_tiled=is_tiled,
is_extend=is_extend,
definition_id=id
).values_list('applies_at_datetime', flat=True).distinct()
return cls.grab_tiled_overlays_from_dates(dates, ids, is_tiled=is_tiled, is_extend=is_extend)
def do_pipeline():
# Cleaning up old files from the database and the disk
DataFileManager.delete_old_files()
OverlayManager.delete_old_files()
# Check for new feedback surveys or comments, and email them to Flaxen
FeedbackHistory.send_feedback_forms()
FeedbackQuestionaire.send_feedback_survey()
#Downloading the latest datafiles for our models. See the appropriate functions
#pl_download/models.py.DataFileManager.get_latest_wave_watch_files() and
#pl_download/models.py.DataFileManager.fetch_new_files() respectively
try:
#Sometimes even though the file downloads this process hangs and fails.
#The try catch is a stop-gap fix so that the pipeline doesn't stop here
#When it fails in that manner the file is downloaded and can be used
wave_watch_files = DataFileManager.get_latest_wave_watch_files()
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
sst_files = DataFileManager.fetch_new_files() # not calling as a task so it runs inline
wind_files = DataFileManager.get_wind_file()
# If no new files were returned, don't plot or tile anything.
try:
#This try catch is also for the wave watch timeout bug
if not wave_watch_files and not sst_files and not wind_files:
print "No New Files Available, Quitting."
return None
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
# get the list of plotting tasks based on the files we just downloaded.
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days()
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4 and pt.args[0] != 6 and pt.args[0] != 7:
# chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
#Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
job = group(item for item in list_of_chains)
print "jobs:"
for each in job:
print each
#and run the group.
result = job.apply_async()
return result
def get_tasks_for_base_plots_for_next_few_days(cls):
""" Grabs the files ID's of NCDF datafiles that haven't been plotted yet. Use this function
to generate a list of unchopped plots that haven't been plotted yet. """
file_ids = []
file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('NCDF')) # OSU ROMS
file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('NCEP')) # Wave Forecast
file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('WIND')) # Wind
file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('T-CLINE')) # T-Cline
file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('HYCOM')) # Hycom - Extended
file_ids = [item for sublist in file_ids for item in sublist] # Unravel lists of lists
return cls.get_tasks_for_base_plots_in_files(file_ids)
def test(ids=None, navy=False, ncep=False):
from pl_download.models import DataFile
from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
from pl_plot.models import OverlayManager as om
print ids
if navy:
print "NAVY HYCOM TEST"
count = 0
if ids:
for id in ids:
datafile = DataFile.objects.get(pk=id)
if datafile.type != 'HYCOM':
continue # Skip over non HYCOM Files
count += 1
print "HYCOM TESTING ", count, " FOR FILE: "
info(datafile)
print "\tTESTING PLOTTER:"
plotter = NavyPlotter(datafile.file.name)
if plotter:
print "\t PLOTTER LOADED SUCCESFULLY"
else:
print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name
print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()
print "\t GENERATING PLOT.... "
print ""
if datafile.file.name.endswith("ssh.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("temp_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
if datafile.file.name.endswith("temp_bot.nc"):
om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
if datafile.file.name.endswith("cur_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("sal_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)
if ncep:
from pl_download.models import DataFileManager as dm
print dm.ww3_download_openDAP()
print "TESTING TASK CREATION"
def test(ids=None, navy=False, ncep=False):
from pl_download.models import DataFile
from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
from pl_plot.models import OverlayManager as om
print ids
if navy:
print "NAVY HYCOM TEST"
count = 0
if ids:
for id in ids:
datafile = DataFile.objects.get(pk=id)
if datafile.type != 'HYCOM':
continue # Skip over non HYCOM Files
count += 1
print "HYCOM TESTING ", count, " FOR FILE: "
info(datafile)
print "\tTESTING PLOTTER:"
plotter = NavyPlotter(datafile.file.name)
if plotter:
print "\t PLOTTER LOADED SUCCESFULLY"
else:
print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name
print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times(
)
print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()
print "\t GENERATING PLOT.... "
print ""
if datafile.file.name.endswith("ssh.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("temp_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
if datafile.file.name.endswith("temp_bot.nc"):
om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
if datafile.file.name.endswith("cur_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
if datafile.file.name.endswith("sal_top.nc"):
om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)
if ncep:
from pl_download.models import DataFileManager as dm
print dm.ww3_download_openDAP()
print "TESTING TASK CREATION"
def do_pipeline():
print "TASKS: before deleting files"
DataFileManager.delete_old_files()
print "TASKS: after deleting Datafiles"
OverlayManager.delete_old_files()
print "TASKS: after deleting Overlays"
wave_watch_files = DataFileManager.get_latest_wave_watch_files()
print "TASKS: after getting wave files"
other_files = DataFileManager.fetch_new_files(
) # not calling as a task so it runs inline
print "TASKS: after getting sst/currents files"
# If no new files were returned, don't plot or tile anything.
if not wave_watch_files and not other_files:
return None
# get the list of plotting tasks based on the files we just downloaded.
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days(
)
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4:
# chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
#Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
job = group(item for item in list_of_chains)
print "jobs:"
for each in job:
print each
#and run the group.
result = job.apply_async()
return result
def download(roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False,
num_dl=None):
from pl_download.models import DataFileManager
ids = []
if roms:
print "DL: Downloading roms files"
roms_ids = []
roms_ids = DataFileManager.download_osu_roms()
print("OSU ROM dl ids:", roms_ids)
ids.append(roms_ids) # Update to a dictonary
if wave:
print "DL: Downloading OSU WW3 files"
wave_ids = []
wave_ids = DataFileManager.get_latest_wave_watch_files()
print("OSU WW3 dl ids:", wave_ids)
ids.append(wave_ids)
if wind:
print "DL: Downloading NAM WIND files"
wind_ids = []
wind_ids = DataFileManager.get_wind_file()
print("NAM Wind dl ids:", wind_ids)
ids.append(wind_ids)
if hycom:
print "DL: Downloading HYCOM files"
print "DL: Number of downloads specified: ", num_dl
hycom_ids = []
hycom_ids = DataFileManager.rtofs_download(count=num_dl)
print("DL: HYCOM dl ids:", hycom_ids)
ids.append(hycom_ids)
if ncep:
print "DL: Downloaind NCEP WW3"
ncep_ids = []
ncep_ids = DataFileManager.ww3_download()
print("NCEP dl ids:", ncep_ids)
ids.append(ncep_ids)
if tcline:
print "DL: Downloading OSU t-cline"
tcline_ids = []
tcline_ids = None
print("NCEP dl ids:", tcline_ids)
ids.append(tcline_ids)
if navy:
print "DL: Downloading Navy Hycom"
navy_ids = []
navy_ids = DataFileManager.navy_hycom_download()
print ("NAVY HYCOM ids: ", navy_ids)
ids.append(navy_ids)
return ids
def get_tasks_for_base_plots_for_next_few_days(cls):
""" Grabs the files ID's of NCDF datafiles that haven't been plotted yet. Use this function
to generate a list of unchopped plots that haven't been plotted yet. """
file_ids = []
file_ids.append(
DataFileManager.get_next_few_datafiles_of_a_type(
'NCDF')) # OSU ROMS
file_ids.append(
DataFileManager.get_next_few_datafiles_of_a_type(
'NCEP')) # Wave Forecast
file_ids.append(
DataFileManager.get_next_few_datafiles_of_a_type('WIND')) # Wind
file_ids.append(
DataFileManager.get_next_few_datafiles_of_a_type(
'T-CLINE')) # T-Cline
file_ids.append(
DataFileManager.get_next_few_datafiles_of_a_type(
'HYCOM')) # Hycom - Extended
file_ids = [item for sublist in file_ids
for item in sublist] # Unravel lists of lists
return cls.get_tasks_for_base_plots_in_files(file_ids)
def get_next_few_days_of_tiled_overlays_for_extended_forecasts(
cls, type, is_tiled=True, is_extend=True):
"""
TODO: Update Docstring for this fucntion
:param type:
:param models:
:param is_tiled:
:param is_extend:
:return: TODO: Find out exactly what this function returns
"""
extend_date = None
if type == 'NCEP':
extend_date = DataFileManager.get_last_forecast_for_osu_ww3()
ids = [settings.OSU_WW3_HI, settings.OSU_WW3_DIR]
elif type == 'NCDF':
extend_date = DataFileManager.get_last_forecast_for_roms()
ids = [settings.OSU_ROMS_SST, settings.OSU_ROMS_SUR_CUR]
else:
print "Wrong type! Returning!"
return -1
dates = []
print "EXTEND DATE: ", extend_date
for id in ids: # Grab the dates based on the extend date found above
dates = Overlay.objects.filter(
applies_at_datetime__gte=extend_date,
is_tiled=is_tiled,
is_extend=is_extend,
definition_id=id).values_list('applies_at_datetime',
flat=True).distinct()
return cls.grab_tiled_overlays_from_dates(dates,
ids,
is_tiled=is_tiled,
is_extend=is_extend)
def get_tasks_for_base_plots_for_next_few_days(cls):
file_ids = [datafile.id for datafile in DataFileManager.get_next_few_days_files_from_db()]
return cls.get_tasks_for_base_plots_in_files(file_ids)
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "SharkEyesCore.settings")
import SharkEyesCore.startup as startup
startup.run()
if sys.argv[-1] == "plot":
from pl_download.models import DataFileManager, DataFile
from pl_plot.models import OverlayManager
from pl_chop.tasks import tile_overlay, tile_wave_watch_overlay
wave = 0
sst = 0
wind = 1
if wave:
wave = DataFileManager.get_latest_wave_watch_files()
wave = DataFile.objects.filter(type='WAVE').latest('model_date')
tiles = []
begin = time.time()
#first entry is day-1 at 12pm
#need to offset 16 to match with sst plot
#NOTE it increments in 1 hour changes
tiles += OverlayManager.make_wave_watch_plot(4, 16, wave[0])
tiles += OverlayManager.make_wave_watch_plot(6, 16, wave[0])
for t in tiles:
tile_wave_watch_overlay(t)
finish = time.time()
totalTime = (finish - begin)/ 60
print "Time taken for Waves = " + str(round(totalTime, 2)) + " minutes"
if sst:
sst = DataFileManager.fetch_new_files()
def do_pipeline():
logging.info('DO_PIPELINE STARTED')
# Cleaning up old files from the database and the disk
print "CLEANING UP - DELETING OLD FILES"
logging.info('CLEANING UP - DELETING OLD DATAFILE FILES')
try:
DataFileManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OLD NETCDF FILES"
logging.error('CLEANING UP - ERROR DELETING DATAFILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD DATAFILE FILES DELETED')
logging.info('CLEANING UP - DELETING OLD OVERLAY FILES')
try:
OverlayManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OVERLAY FILES"
logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED')
# Check for new feedback surveys or comments, and email them to Flaxen
print "SENDING OUT FEEDBACK"
logging.info('SENDING OUT FEEDBACK')
try:
FeedbackHistory.send_feedback_forms()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK"
logging.error('ERROR SENDING FEEDBACK FORMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('FEEDBACK FORMS SENT')
logging.info('SENDING OUT SURVEY')
try:
FeedbackQuestionaire.send_feedback_survey()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK SURVEY"
logging.error('ERROR SENDING FEEDBACK SURVEY')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('SURVEYS Sent')
print "DOWNLOADING OSU ROMS FILES"
logging.info('DOWNLOADING OSU ROMS')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
sst_files = DataFileManager.download_osu_roms()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD OSU ROMS FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
if settings.WW3_OPENDAP is True:
print "DOWNLOADING NCEP FILES VIA OPENDAP"
logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download_openDAP()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP"
logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY')
else:
print "DOWNLOADING UCAR NCEP FILES"
logging.info('DOWNLOADING UCAR NCEP WW3')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES"
logging.error('ERROR DOWNLOADING UCAR NCEP WW3')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU WW3 DOWNLOADED SUCCESFULLY')
if settings.EXTEND:
print "DOWNLOADING NAVY HYCOM"
logging.info('DOWNLOADING NAVY HYCOM')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
hycom_files = DataFileManager.navy_hycom_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NAVY HYCOM FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING WIND FILES"
logging.info('DOWNLOADING NAM WIND')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wind_files = DataFileManager.get_wind_file()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD WIND FILE"
logging.error('ERROR DOWNLOADING NAM WINDS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NAM WINDS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING TCLINE FILES"
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
tcline_files = DataFileManager.download_tcline()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD TCLINE FILE"
traceback.print_exc(file=sys.stdout)
print '-' * 60
try:
#This try catch is also for the wave watch timeout bug
if not wave_watch_files and not sst_files and not wind_files \
and not hycom_files and not tcline_files:
print "No New Files Available, Quitting."
return None
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
# Get the list of plotting tasks based on the files we just downloaded.
logging.info('GENERATING TASK LIST')
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days()
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4 and pt.args[0] != 6:
# Chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
# Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
logging.info('TASK LIST GENERATED')
job = group(item for item in list_of_chains)
print "PIPELINE: JOBS: "
for each in job:
print each
logging.info('APPLY JOBS')
result = job.apply_async() # Run the group.
logging.info('JOBS APPLIED')
return result
def download(roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False,
num_dl=None):
from pl_download.models import DataFileManager
ids = []
if roms:
print "DL: Downloading roms files"
roms_ids = []
roms_ids = DataFileManager.download_osu_roms()
print("OSU ROM dl ids:", roms_ids)
ids.append(roms_ids) # Update to a dictonary
if wave:
print "DL: Downloading OSU WW3 files"
wave_ids = []
wave_ids = DataFileManager.get_latest_wave_watch_files()
print("OSU WW3 dl ids:", wave_ids)
ids.append(wave_ids)
if wind:
print "DL: Downloading NAM WIND files"
wind_ids = []
wind_ids = DataFileManager.get_wind_file()
print("NAM Wind dl ids:", wind_ids)
ids.append(wind_ids)
if hycom:
print "DL: Downloading HYCOM files"
print "DL: Number of downloads specified: ", num_dl
hycom_ids = []
hycom_ids = DataFileManager.rtofs_download(count=num_dl)
print("DL: HYCOM dl ids:", hycom_ids)
ids.append(hycom_ids)
if ncep:
ncep_ids = []
if settings.WW3_OPENDAP:
print "DL: Downloaind NCEP WW3 via OpenDAP"
ncep_ids = DataFileManager.ww3_download_openDAP()
else:
print "DL: Downloaind NCEP WW3"
ncep_ids = DataFileManager.ww3_download()
print("NCEP dl ids:", ncep_ids)
ids.append(ncep_ids)
if tcline:
print "DL: Downloading OSU t-cline"
tcline_ids = []
tcline_ids = None
print("NCEP dl ids:", tcline_ids)
ids.append(tcline_ids)
if navy:
print "DL: Downloading Navy Hycom"
navy_ids = []
navy_ids = DataFileManager.navy_hycom_download()
print ("NAVY HYCOM ids: ", navy_ids)
ids.append(navy_ids)
return ids
def testWaveWatchDownload(self):
print "Running Wave Watch Download Test: "
result = DataFileManager.get_latest_wave_watch_files()
self.assertIsNotNone(result)
def testFetchingFiles(self):
print "Running Currents & SS Temperature Download Test: "
result = DataFileManager.fetch_new_files()
self.assertIsNotNone(result)
def do_pipeline():
logging.info('DO_PIPELINE STARTED')
# Cleaning up old files from the database and the disk
print "CLEANING UP - DELETING OLD FILES"
logging.info('CLEANING UP - DELETING OLD DATAFILE FILES')
try:
DataFileManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OLD NETCDF FILES"
logging.error('CLEANING UP - ERROR DELETING DATAFILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD DATAFILE FILES DELETED')
logging.info('CLEANING UP - DELETING OLD OVERLAY FILES')
try:
OverlayManager.delete_old_files()
except Exception:
print '-' * 60
print "COULD NOT DELETE OVERLAY FILES"
logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED')
# Check for new feedback surveys or comments, and email them to Flaxen
print "SENDING OUT FEEDBACK"
logging.info('SENDING OUT FEEDBACK')
try:
FeedbackHistory.send_feedback_forms()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK"
logging.error('ERROR SENDING FEEDBACK FORMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('FEEDBACK FORMS SENT')
logging.info('SENDING OUT SURVEY')
try:
FeedbackQuestionaire.send_feedback_survey()
except Exception:
print '-' * 60
print "COULD NOT SEND FEEDBACK SURVEY"
logging.error('ERROR SENDING FEEDBACK SURVEY')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('SURVEYS Sent')
print "DOWNLOADING OSU ROMS FILES"
logging.info('DOWNLOADING OSU ROMS')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
sst_files = DataFileManager.download_osu_roms()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD OSU ROMS FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
if settings.WW3_OPENDAP is True:
print "DOWNLOADING NCEP FILES VIA OPENDAP"
logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download_openDAP()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP"
logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY')
else:
print "DOWNLOADING UCAR NCEP FILES"
logging.info('DOWNLOADING UCAR NCEP WW3')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wave_watch_files = DataFileManager.ww3_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES"
logging.error('ERROR DOWNLOADING UCAR NCEP WW3')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU WW3 DOWNLOADED SUCCESFULLY')
if settings.EXTEND:
print "DOWNLOADING NAVY HYCOM"
logging.info('DOWNLOADING NAVY HYCOM')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
hycom_files = DataFileManager.navy_hycom_download()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD NAVY HYCOM FILES"
logging.error('ERROR DOWNLOADING OSU ROMS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING WIND FILES"
logging.info('DOWNLOADING NAM WIND')
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
wind_files = DataFileManager.get_wind_file()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD WIND FILE"
logging.error('ERROR DOWNLOADING NAM WINDS')
traceback.print_exc(file=sys.stdout)
print '-' * 60
logging.info('NAM WINDS DOWNLOADED SUCCESFULLY')
print "DOWNLOADING TCLINE FILES"
try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
tcline_files = DataFileManager.download_tcline()
except Exception:
print '-' * 60
print "COULD NOT DOWNLOAD TCLINE FILE"
traceback.print_exc(file=sys.stdout)
print '-' * 60
try:
#This try catch is also for the wave watch timeout bug
if not wave_watch_files and not sst_files and not wind_files \
and not hycom_files and not tcline_files:
print "No New Files Available, Quitting."
return None
except Exception:
print '-' * 60
traceback.print_exc(file=sys.stdout)
print '-' * 60
# Get the list of plotting tasks based on the files we just downloaded.
logging.info('GENERATING TASK LIST')
plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days(
)
list_of_chains = []
for pt in plot_task_list:
if pt.args[0] != 4 and pt.args[0] != 6:
# Chaining passes the result of first function to second function
list_of_chains.append(chain(pt, tile_overlay.s()))
else:
# Use the Wavewatch tiler for Wavewatch files
list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
logging.info('TASK LIST GENERATED')
job = group(item for item in list_of_chains)
print "PIPELINE: JOBS: "
for each in job:
print each
logging.info('APPLY JOBS')
result = job.apply_async() # Run the group.
logging.info('JOBS APPLIED')
return result
def plot_latest(num_plots=DEF_NUM_PLOTS, tile=DEF_TILE_FLAG, full_roms=DEF_FULL_ROMS_FLAG, date='latest',
roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False):
# num_plots = The number of plots you want for each file - save time!
# Pull the latest files from the database and plot those
"""
Using `latest` grabs all plots into the future. Similar to do_pipeline()
"""
from pl_download.models import DataFile as df
from pl_download.models import DataFileManager as dm
from datetime import datetime
if verbose > 0:
print "Date span request: ", date
# Today
if date == 'today':
today = datetime.now().date()
print "Printing all datafiles with date of today"
elif date == 'all':
print "Printing all datafiles"
pass
elif date == 'latest':
print "Printing datafiles that start with PAST_FILES_TO_DISPALY until whats returned from \n"
print "get_next_few_days_files_from_db(days=x)"
pass
elif date is None:
print "Printing datafiles that start with PAST_FILES_TO_DISPALY until whats returned from \n"
print "get_next_few_days_files_from_db(days=x)"
date = 'latest'
else:
print "Wrong date paramater. Please use - 'today', 'all', or 'latest'"
return
if roms:
roms = []
ids = []
if date == "today":
ids = df.objects.filter(type='NCDF').get(model_date=today)
elif date == "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='NCDF')
elif date == "all":
ids = df.objects.all().filter(type = "NCDF")
print ids
ids = [id.id for id in ids] # Unwrap ids
plot( ids, roms=True, num_plots=num_plots, tile=tile, full_roms=full_roms )
if wave:
waves = []
ids = []
if date == "today":
ids = df.objects.filter(type='WAVE').get(model_date=today)
elif date == "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='WAVE')
elif date == "all":
ids = df.objects.all().filter(type = "WAVE")
ids = [id.id for id in ids] # Unwrap ids
plot( ids, wave=True, num_plots=num_plots, tile=tile )
if wind:
winds = []
ids = []
if date is "today":
ids = df.objects.filter(type='WIND').get(model_date=today)
elif date is "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='WIND')
elif date is "all":
ids = df.objects.all().filter(type = "WIND")
ids = [id.id for id in ids] # Unwrap ids
plot(ids, wind=True, num_plots=num_plots, tile=tile )
if hycom:
hycoms = []
if date == "today":
ids = df.objects.filter(type='HYCOM').get(model_date=today)
elif date == "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='HYCOM')
elif date == "all":
ids = df.objects.all().filter(type = "HYCOM")
ids = [id.id for id in ids] # Unwrap ids
plot( ids, hycom=True, num_plots=num_plots, tile=tile )
if ncep:
nceps = []
ids = []
if date is "today":
ids = df.objects.filter(type='NCEP_WW3').get(model_date=today)
elif date is "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='NCEP_WW3')
elif date is "all":
ids = df.objects.all().filter(type = "NCEP_WW3")
ids = [id.id for id in ids] # Unwrap ids
plot( ids, ncep=True, num_plots=num_plots, tile=tile )
if tcline:
tcline = []
ids = []
if date is "today":
ids = df.objects.filter(type='T-CLINE').get(model_date=today)
elif date is "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='T-CLINE')
elif date is "all":
ids = df.objects.all().filter(type = "T-CLINE")
ids = [id.id for id in ids] # Unwrap ids
plot( ids, tcline=True, num_plots=num_plots, tile=tile )
if navy:
navys = []
ids = []
if date is "today":
ids = df.objects.filter(type='HYCOM').get(model_date=today)
elif date is "latest":
ids = dm.get_next_few_datafiles_of_a_type(type ='HYCOM')
elif date is "all":
ids = df.objects.all().filter(type = "HYCOM")
ids = [id.id for id in ids] # Unwrap ids
plot( ids, navy=True, num_plots=num_plots, tile=tile )
print "MANAGE.PY: FINISH TEST"
