def get_next_few_days_of_tiled_overlays_for_extended_forecasts(cls, type, is_tiled=True, is_extend=True): """ TODO: Update Docstring for this fucntion :param type: :param models: :param is_tiled: :param is_extend: :return: TODO: Find out exactly what this function returns """ extend_date = None if type == 'NCEP': extend_date = DataFileManager.get_last_forecast_for_osu_ww3() ids = [settings.OSU_WW3_HI, settings.OSU_WW3_DIR] elif type == 'NCDF': extend_date = DataFileManager.get_last_forecast_for_roms() ids = [settings.OSU_ROMS_SST, settings.OSU_ROMS_SUR_CUR] else: print "Wrong type! Returning!" return -1 dates = [] print "EXTEND DATE: ", extend_date for id in ids: # Grab the dates based on the extend date found above dates = Overlay.objects.filter(applies_at_datetime__gte=extend_date, is_tiled=is_tiled, is_extend=is_extend, definition_id=id ).values_list('applies_at_datetime', flat=True).distinct() return cls.grab_tiled_overlays_from_dates(dates, ids, is_tiled=is_tiled, is_extend=is_extend)
def do_pipeline(): # Cleaning up old files from the database and the disk DataFileManager.delete_old_files() OverlayManager.delete_old_files() # Check for new feedback surveys or comments, and email them to Flaxen FeedbackHistory.send_feedback_forms() FeedbackQuestionaire.send_feedback_survey() #Downloading the latest datafiles for our models. See the appropriate functions #pl_download/models.py.DataFileManager.get_latest_wave_watch_files() and #pl_download/models.py.DataFileManager.fetch_new_files() respectively try: #Sometimes even though the file downloads this process hangs and fails. #The try catch is a stop-gap fix so that the pipeline doesn't stop here #When it fails in that manner the file is downloaded and can be used wave_watch_files = DataFileManager.get_latest_wave_watch_files() except Exception: print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 sst_files = DataFileManager.fetch_new_files() # not calling as a task so it runs inline wind_files = DataFileManager.get_wind_file() # If no new files were returned, don't plot or tile anything. try: #This try catch is also for the wave watch timeout bug if not wave_watch_files and not sst_files and not wind_files: print "No New Files Available, Quitting." return None except Exception: print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 # get the list of plotting tasks based on the files we just downloaded. plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days() list_of_chains = [] for pt in plot_task_list: if pt.args[0] != 4 and pt.args[0] != 6 and pt.args[0] != 7: # chaining passes the result of first function to second function list_of_chains.append(chain(pt, tile_overlay.s())) else: #Use the Wavewatch tiler for Wavewatch files list_of_chains.append(chain(pt, tile_wave_watch_overlay.s())) job = group(item for item in list_of_chains) print "jobs:" for each in job: print each #and run the group. result = job.apply_async() return result
def get_tasks_for_base_plots_for_next_few_days(cls): """ Grabs the files ID's of NCDF datafiles that haven't been plotted yet. Use this function to generate a list of unchopped plots that haven't been plotted yet. """ file_ids = [] file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('NCDF')) # OSU ROMS file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('NCEP')) # Wave Forecast file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('WIND')) # Wind file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('T-CLINE')) # T-Cline file_ids.append(DataFileManager.get_next_few_datafiles_of_a_type('HYCOM')) # Hycom - Extended file_ids = [item for sublist in file_ids for item in sublist] # Unravel lists of lists return cls.get_tasks_for_base_plots_in_files(file_ids)
def test(ids=None, navy=False, ncep=False): from pl_download.models import DataFile from pl_plot.plotter import NavyPlotter, NcepWW3Plotter from pl_plot.models import OverlayManager as om print ids if navy: print "NAVY HYCOM TEST" count = 0 if ids: for id in ids: datafile = DataFile.objects.get(pk=id) if datafile.type != 'HYCOM': continue # Skip over non HYCOM Files count += 1 print "HYCOM TESTING ", count, " FOR FILE: " info(datafile) print "\tTESTING PLOTTER:" plotter = NavyPlotter(datafile.file.name) if plotter: print "\t PLOTTER LOADED SUCCESFULLY" else: print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times() print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index() print "\t GENERATING PLOT.... " print "" if datafile.file.name.endswith("ssh.nc"): om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id) if datafile.file.name.endswith("temp_top.nc"): om.make_plot(settings.NAVY_HYCOM_SST, 0, id) if datafile.file.name.endswith("temp_bot.nc"): om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id) if datafile.file.name.endswith("cur_top.nc"): om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id) if datafile.file.name.endswith("sal_top.nc"): om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id) if ncep: from pl_download.models import DataFileManager as dm print dm.ww3_download_openDAP() print "TESTING TASK CREATION"
def test(ids=None, navy=False, ncep=False): from pl_download.models import DataFile from pl_plot.plotter import NavyPlotter, NcepWW3Plotter from pl_plot.models import OverlayManager as om print ids if navy: print "NAVY HYCOM TEST" count = 0 if ids: for id in ids: datafile = DataFile.objects.get(pk=id) if datafile.type != 'HYCOM': continue # Skip over non HYCOM Files count += 1 print "HYCOM TESTING ", count, " FOR FILE: " info(datafile) print "\tTESTING PLOTTER:" plotter = NavyPlotter(datafile.file.name) if plotter: print "\t PLOTTER LOADED SUCCESFULLY" else: print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times( ) print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index() print "\t GENERATING PLOT.... " print "" if datafile.file.name.endswith("ssh.nc"): om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id) if datafile.file.name.endswith("temp_top.nc"): om.make_plot(settings.NAVY_HYCOM_SST, 0, id) if datafile.file.name.endswith("temp_bot.nc"): om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id) if datafile.file.name.endswith("cur_top.nc"): om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id) if datafile.file.name.endswith("sal_top.nc"): om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id) if ncep: from pl_download.models import DataFileManager as dm print dm.ww3_download_openDAP() print "TESTING TASK CREATION"
def do_pipeline(): print "TASKS: before deleting files" DataFileManager.delete_old_files() print "TASKS: after deleting Datafiles" OverlayManager.delete_old_files() print "TASKS: after deleting Overlays" wave_watch_files = DataFileManager.get_latest_wave_watch_files() print "TASKS: after getting wave files" other_files = DataFileManager.fetch_new_files( ) # not calling as a task so it runs inline print "TASKS: after getting sst/currents files" # If no new files were returned, don't plot or tile anything. if not wave_watch_files and not other_files: return None # get the list of plotting tasks based on the files we just downloaded. plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days( ) list_of_chains = [] for pt in plot_task_list: if pt.args[0] != 4: # chaining passes the result of first function to second function list_of_chains.append(chain(pt, tile_overlay.s())) else: #Use the Wavewatch tiler for Wavewatch files list_of_chains.append(chain(pt, tile_wave_watch_overlay.s())) job = group(item for item in list_of_chains) print "jobs:" for each in job: print each #and run the group. result = job.apply_async() return result
def download(roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False, num_dl=None): from pl_download.models import DataFileManager ids = [] if roms: print "DL: Downloading roms files" roms_ids = [] roms_ids = DataFileManager.download_osu_roms() print("OSU ROM dl ids:", roms_ids) ids.append(roms_ids) # Update to a dictonary if wave: print "DL: Downloading OSU WW3 files" wave_ids = [] wave_ids = DataFileManager.get_latest_wave_watch_files() print("OSU WW3 dl ids:", wave_ids) ids.append(wave_ids) if wind: print "DL: Downloading NAM WIND files" wind_ids = [] wind_ids = DataFileManager.get_wind_file() print("NAM Wind dl ids:", wind_ids) ids.append(wind_ids) if hycom: print "DL: Downloading HYCOM files" print "DL: Number of downloads specified: ", num_dl hycom_ids = [] hycom_ids = DataFileManager.rtofs_download(count=num_dl) print("DL: HYCOM dl ids:", hycom_ids) ids.append(hycom_ids) if ncep: print "DL: Downloaind NCEP WW3" ncep_ids = [] ncep_ids = DataFileManager.ww3_download() print("NCEP dl ids:", ncep_ids) ids.append(ncep_ids) if tcline: print "DL: Downloading OSU t-cline" tcline_ids = [] tcline_ids = None print("NCEP dl ids:", tcline_ids) ids.append(tcline_ids) if navy: print "DL: Downloading Navy Hycom" navy_ids = [] navy_ids = DataFileManager.navy_hycom_download() print ("NAVY HYCOM ids: ", navy_ids) ids.append(navy_ids) return ids
def get_tasks_for_base_plots_for_next_few_days(cls): """ Grabs the files ID's of NCDF datafiles that haven't been plotted yet. Use this function to generate a list of unchopped plots that haven't been plotted yet. """ file_ids = [] file_ids.append( DataFileManager.get_next_few_datafiles_of_a_type( 'NCDF')) # OSU ROMS file_ids.append( DataFileManager.get_next_few_datafiles_of_a_type( 'NCEP')) # Wave Forecast file_ids.append( DataFileManager.get_next_few_datafiles_of_a_type('WIND')) # Wind file_ids.append( DataFileManager.get_next_few_datafiles_of_a_type( 'T-CLINE')) # T-Cline file_ids.append( DataFileManager.get_next_few_datafiles_of_a_type( 'HYCOM')) # Hycom - Extended file_ids = [item for sublist in file_ids for item in sublist] # Unravel lists of lists return cls.get_tasks_for_base_plots_in_files(file_ids)
def get_next_few_days_of_tiled_overlays_for_extended_forecasts( cls, type, is_tiled=True, is_extend=True): """ TODO: Update Docstring for this fucntion :param type: :param models: :param is_tiled: :param is_extend: :return: TODO: Find out exactly what this function returns """ extend_date = None if type == 'NCEP': extend_date = DataFileManager.get_last_forecast_for_osu_ww3() ids = [settings.OSU_WW3_HI, settings.OSU_WW3_DIR] elif type == 'NCDF': extend_date = DataFileManager.get_last_forecast_for_roms() ids = [settings.OSU_ROMS_SST, settings.OSU_ROMS_SUR_CUR] else: print "Wrong type! Returning!" return -1 dates = [] print "EXTEND DATE: ", extend_date for id in ids: # Grab the dates based on the extend date found above dates = Overlay.objects.filter( applies_at_datetime__gte=extend_date, is_tiled=is_tiled, is_extend=is_extend, definition_id=id).values_list('applies_at_datetime', flat=True).distinct() return cls.grab_tiled_overlays_from_dates(dates, ids, is_tiled=is_tiled, is_extend=is_extend)
def get_tasks_for_base_plots_for_next_few_days(cls): file_ids = [datafile.id for datafile in DataFileManager.get_next_few_days_files_from_db()] return cls.get_tasks_for_base_plots_in_files(file_ids)
if __name__ == "__main__": os.environ.setdefault("DJANGO_SETTINGS_MODULE", "SharkEyesCore.settings") import SharkEyesCore.startup as startup startup.run() if sys.argv[-1] == "plot": from pl_download.models import DataFileManager, DataFile from pl_plot.models import OverlayManager from pl_chop.tasks import tile_overlay, tile_wave_watch_overlay wave = 0 sst = 0 wind = 1 if wave: wave = DataFileManager.get_latest_wave_watch_files() wave = DataFile.objects.filter(type='WAVE').latest('model_date') tiles = [] begin = time.time() #first entry is day-1 at 12pm #need to offset 16 to match with sst plot #NOTE it increments in 1 hour changes tiles += OverlayManager.make_wave_watch_plot(4, 16, wave[0]) tiles += OverlayManager.make_wave_watch_plot(6, 16, wave[0]) for t in tiles: tile_wave_watch_overlay(t) finish = time.time() totalTime = (finish - begin)/ 60 print "Time taken for Waves = " + str(round(totalTime, 2)) + " minutes" if sst: sst = DataFileManager.fetch_new_files()
def do_pipeline(): logging.info('DO_PIPELINE STARTED') # Cleaning up old files from the database and the disk print "CLEANING UP - DELETING OLD FILES" logging.info('CLEANING UP - DELETING OLD DATAFILE FILES') try: DataFileManager.delete_old_files() except Exception: print '-' * 60 print "COULD NOT DELETE OLD NETCDF FILES" logging.error('CLEANING UP - ERROR DELETING DATAFILES') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('CLEANING UP - OLD DATAFILE FILES DELETED') logging.info('CLEANING UP - DELETING OLD OVERLAY FILES') try: OverlayManager.delete_old_files() except Exception: print '-' * 60 print "COULD NOT DELETE OVERLAY FILES" logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED') # Check for new feedback surveys or comments, and email them to Flaxen print "SENDING OUT FEEDBACK" logging.info('SENDING OUT FEEDBACK') try: FeedbackHistory.send_feedback_forms() except Exception: print '-' * 60 print "COULD NOT SEND FEEDBACK" logging.error('ERROR SENDING FEEDBACK FORMS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('FEEDBACK FORMS SENT') logging.info('SENDING OUT SURVEY') try: FeedbackQuestionaire.send_feedback_survey() except Exception: print '-' * 60 print "COULD NOT SEND FEEDBACK SURVEY" logging.error('ERROR SENDING FEEDBACK SURVEY') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('SURVEYS Sent') print "DOWNLOADING OSU ROMS FILES" logging.info('DOWNLOADING OSU ROMS') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached sst_files = DataFileManager.download_osu_roms() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD OSU ROMS FILES" logging.error('ERROR DOWNLOADING OSU ROMS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('OSU ROMS DOWNLOADED SUCCESFULLY') if settings.WW3_OPENDAP is True: print "DOWNLOADING NCEP FILES VIA OPENDAP" logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached wave_watch_files = DataFileManager.ww3_download_openDAP() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP" logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY') else: print "DOWNLOADING UCAR NCEP FILES" logging.info('DOWNLOADING UCAR NCEP WW3') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached wave_watch_files = DataFileManager.ww3_download() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES" logging.error('ERROR DOWNLOADING UCAR NCEP WW3') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('OSU WW3 DOWNLOADED SUCCESFULLY') if settings.EXTEND: print "DOWNLOADING NAVY HYCOM" logging.info('DOWNLOADING NAVY HYCOM') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached hycom_files = DataFileManager.navy_hycom_download() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD NAVY HYCOM FILES" logging.error('ERROR DOWNLOADING OSU ROMS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('OSU ROMS DOWNLOADED SUCCESFULLY') print "DOWNLOADING WIND FILES" logging.info('DOWNLOADING NAM WIND') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached wind_files = DataFileManager.get_wind_file() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD WIND FILE" logging.error('ERROR DOWNLOADING NAM WINDS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('NAM WINDS DOWNLOADED SUCCESFULLY') print "DOWNLOADING TCLINE FILES" try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached tcline_files = DataFileManager.download_tcline() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD TCLINE FILE" traceback.print_exc(file=sys.stdout) print '-' * 60 try: #This try catch is also for the wave watch timeout bug if not wave_watch_files and not sst_files and not wind_files \ and not hycom_files and not tcline_files: print "No New Files Available, Quitting." return None except Exception: print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 # Get the list of plotting tasks based on the files we just downloaded. logging.info('GENERATING TASK LIST') plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days() list_of_chains = [] for pt in plot_task_list: if pt.args[0] != 4 and pt.args[0] != 6: # Chaining passes the result of first function to second function list_of_chains.append(chain(pt, tile_overlay.s())) else: # Use the Wavewatch tiler for Wavewatch files list_of_chains.append(chain(pt, tile_wave_watch_overlay.s())) logging.info('TASK LIST GENERATED') job = group(item for item in list_of_chains) print "PIPELINE: JOBS: " for each in job: print each logging.info('APPLY JOBS') result = job.apply_async() # Run the group. logging.info('JOBS APPLIED') return result
def download(roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False, num_dl=None): from pl_download.models import DataFileManager ids = [] if roms: print "DL: Downloading roms files" roms_ids = [] roms_ids = DataFileManager.download_osu_roms() print("OSU ROM dl ids:", roms_ids) ids.append(roms_ids) # Update to a dictonary if wave: print "DL: Downloading OSU WW3 files" wave_ids = [] wave_ids = DataFileManager.get_latest_wave_watch_files() print("OSU WW3 dl ids:", wave_ids) ids.append(wave_ids) if wind: print "DL: Downloading NAM WIND files" wind_ids = [] wind_ids = DataFileManager.get_wind_file() print("NAM Wind dl ids:", wind_ids) ids.append(wind_ids) if hycom: print "DL: Downloading HYCOM files" print "DL: Number of downloads specified: ", num_dl hycom_ids = [] hycom_ids = DataFileManager.rtofs_download(count=num_dl) print("DL: HYCOM dl ids:", hycom_ids) ids.append(hycom_ids) if ncep: ncep_ids = [] if settings.WW3_OPENDAP: print "DL: Downloaind NCEP WW3 via OpenDAP" ncep_ids = DataFileManager.ww3_download_openDAP() else: print "DL: Downloaind NCEP WW3" ncep_ids = DataFileManager.ww3_download() print("NCEP dl ids:", ncep_ids) ids.append(ncep_ids) if tcline: print "DL: Downloading OSU t-cline" tcline_ids = [] tcline_ids = None print("NCEP dl ids:", tcline_ids) ids.append(tcline_ids) if navy: print "DL: Downloading Navy Hycom" navy_ids = [] navy_ids = DataFileManager.navy_hycom_download() print ("NAVY HYCOM ids: ", navy_ids) ids.append(navy_ids) return ids
def testWaveWatchDownload(self): print "Running Wave Watch Download Test: " result = DataFileManager.get_latest_wave_watch_files() self.assertIsNotNone(result)
def testFetchingFiles(self): print "Running Currents & SS Temperature Download Test: " result = DataFileManager.fetch_new_files() self.assertIsNotNone(result)
def do_pipeline(): logging.info('DO_PIPELINE STARTED') # Cleaning up old files from the database and the disk print "CLEANING UP - DELETING OLD FILES" logging.info('CLEANING UP - DELETING OLD DATAFILE FILES') try: DataFileManager.delete_old_files() except Exception: print '-' * 60 print "COULD NOT DELETE OLD NETCDF FILES" logging.error('CLEANING UP - ERROR DELETING DATAFILES') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('CLEANING UP - OLD DATAFILE FILES DELETED') logging.info('CLEANING UP - DELETING OLD OVERLAY FILES') try: OverlayManager.delete_old_files() except Exception: print '-' * 60 print "COULD NOT DELETE OVERLAY FILES" logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED') # Check for new feedback surveys or comments, and email them to Flaxen print "SENDING OUT FEEDBACK" logging.info('SENDING OUT FEEDBACK') try: FeedbackHistory.send_feedback_forms() except Exception: print '-' * 60 print "COULD NOT SEND FEEDBACK" logging.error('ERROR SENDING FEEDBACK FORMS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('FEEDBACK FORMS SENT') logging.info('SENDING OUT SURVEY') try: FeedbackQuestionaire.send_feedback_survey() except Exception: print '-' * 60 print "COULD NOT SEND FEEDBACK SURVEY" logging.error('ERROR SENDING FEEDBACK SURVEY') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('SURVEYS Sent') print "DOWNLOADING OSU ROMS FILES" logging.info('DOWNLOADING OSU ROMS') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached sst_files = DataFileManager.download_osu_roms() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD OSU ROMS FILES" logging.error('ERROR DOWNLOADING OSU ROMS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('OSU ROMS DOWNLOADED SUCCESFULLY') if settings.WW3_OPENDAP is True: print "DOWNLOADING NCEP FILES VIA OPENDAP" logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached wave_watch_files = DataFileManager.ww3_download_openDAP() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP" logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY') else: print "DOWNLOADING UCAR NCEP FILES" logging.info('DOWNLOADING UCAR NCEP WW3') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached wave_watch_files = DataFileManager.ww3_download() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES" logging.error('ERROR DOWNLOADING UCAR NCEP WW3') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('OSU WW3 DOWNLOADED SUCCESFULLY') if settings.EXTEND: print "DOWNLOADING NAVY HYCOM" logging.info('DOWNLOADING NAVY HYCOM') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached hycom_files = DataFileManager.navy_hycom_download() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD NAVY HYCOM FILES" logging.error('ERROR DOWNLOADING OSU ROMS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('OSU ROMS DOWNLOADED SUCCESFULLY') print "DOWNLOADING WIND FILES" logging.info('DOWNLOADING NAM WIND') try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached wind_files = DataFileManager.get_wind_file() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD WIND FILE" logging.error('ERROR DOWNLOADING NAM WINDS') traceback.print_exc(file=sys.stdout) print '-' * 60 logging.info('NAM WINDS DOWNLOADED SUCCESFULLY') print "DOWNLOADING TCLINE FILES" try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached tcline_files = DataFileManager.download_tcline() except Exception: print '-' * 60 print "COULD NOT DOWNLOAD TCLINE FILE" traceback.print_exc(file=sys.stdout) print '-' * 60 try: #This try catch is also for the wave watch timeout bug if not wave_watch_files and not sst_files and not wind_files \ and not hycom_files and not tcline_files: print "No New Files Available, Quitting." return None except Exception: print '-' * 60 traceback.print_exc(file=sys.stdout) print '-' * 60 # Get the list of plotting tasks based on the files we just downloaded. logging.info('GENERATING TASK LIST') plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days( ) list_of_chains = [] for pt in plot_task_list: if pt.args[0] != 4 and pt.args[0] != 6: # Chaining passes the result of first function to second function list_of_chains.append(chain(pt, tile_overlay.s())) else: # Use the Wavewatch tiler for Wavewatch files list_of_chains.append(chain(pt, tile_wave_watch_overlay.s())) logging.info('TASK LIST GENERATED') job = group(item for item in list_of_chains) print "PIPELINE: JOBS: " for each in job: print each logging.info('APPLY JOBS') result = job.apply_async() # Run the group. logging.info('JOBS APPLIED') return result
def plot_latest(num_plots=DEF_NUM_PLOTS, tile=DEF_TILE_FLAG, full_roms=DEF_FULL_ROMS_FLAG, date='latest', roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False): # num_plots = The number of plots you want for each file - save time! # Pull the latest files from the database and plot those """ Using `latest` grabs all plots into the future. Similar to do_pipeline() """ from pl_download.models import DataFile as df from pl_download.models import DataFileManager as dm from datetime import datetime if verbose > 0: print "Date span request: ", date # Today if date == 'today': today = datetime.now().date() print "Printing all datafiles with date of today" elif date == 'all': print "Printing all datafiles" pass elif date == 'latest': print "Printing datafiles that start with PAST_FILES_TO_DISPALY until whats returned from \n" print "get_next_few_days_files_from_db(days=x)" pass elif date is None: print "Printing datafiles that start with PAST_FILES_TO_DISPALY until whats returned from \n" print "get_next_few_days_files_from_db(days=x)" date = 'latest' else: print "Wrong date paramater. Please use - 'today', 'all', or 'latest'" return if roms: roms = [] ids = [] if date == "today": ids = df.objects.filter(type='NCDF').get(model_date=today) elif date == "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='NCDF') elif date == "all": ids = df.objects.all().filter(type = "NCDF") print ids ids = [id.id for id in ids] # Unwrap ids plot( ids, roms=True, num_plots=num_plots, tile=tile, full_roms=full_roms ) if wave: waves = [] ids = [] if date == "today": ids = df.objects.filter(type='WAVE').get(model_date=today) elif date == "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='WAVE') elif date == "all": ids = df.objects.all().filter(type = "WAVE") ids = [id.id for id in ids] # Unwrap ids plot( ids, wave=True, num_plots=num_plots, tile=tile ) if wind: winds = [] ids = [] if date is "today": ids = df.objects.filter(type='WIND').get(model_date=today) elif date is "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='WIND') elif date is "all": ids = df.objects.all().filter(type = "WIND") ids = [id.id for id in ids] # Unwrap ids plot(ids, wind=True, num_plots=num_plots, tile=tile ) if hycom: hycoms = [] if date == "today": ids = df.objects.filter(type='HYCOM').get(model_date=today) elif date == "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='HYCOM') elif date == "all": ids = df.objects.all().filter(type = "HYCOM") ids = [id.id for id in ids] # Unwrap ids plot( ids, hycom=True, num_plots=num_plots, tile=tile ) if ncep: nceps = [] ids = [] if date is "today": ids = df.objects.filter(type='NCEP_WW3').get(model_date=today) elif date is "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='NCEP_WW3') elif date is "all": ids = df.objects.all().filter(type = "NCEP_WW3") ids = [id.id for id in ids] # Unwrap ids plot( ids, ncep=True, num_plots=num_plots, tile=tile ) if tcline: tcline = [] ids = [] if date is "today": ids = df.objects.filter(type='T-CLINE').get(model_date=today) elif date is "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='T-CLINE') elif date is "all": ids = df.objects.all().filter(type = "T-CLINE") ids = [id.id for id in ids] # Unwrap ids plot( ids, tcline=True, num_plots=num_plots, tile=tile ) if navy: navys = [] ids = [] if date is "today": ids = df.objects.filter(type='HYCOM').get(model_date=today) elif date is "latest": ids = dm.get_next_few_datafiles_of_a_type(type ='HYCOM') elif date is "all": ids = df.objects.all().filter(type = "HYCOM") ids = [id.id for id in ids] # Unwrap ids plot( ids, navy=True, num_plots=num_plots, tile=tile ) print "MANAGE.PY: FINISH TEST"