예제 #1
0
파일: tasks.py 프로젝트: seacast/SharkEyes
def do_pipeline():
    # Cleaning up old files from the database and the disk
    DataFileManager.delete_old_files()
    OverlayManager.delete_old_files()

    # Check for new feedback surveys or comments, and email them to Flaxen
    FeedbackHistory.send_feedback_forms()
    FeedbackQuestionaire.send_feedback_survey()

    #Downloading the latest datafiles for our models. See the appropriate functions
    #pl_download/models.py.DataFileManager.get_latest_wave_watch_files() and
    #pl_download/models.py.DataFileManager.fetch_new_files() respectively
    try:
        #Sometimes even though the file downloads this process hangs and fails.
        #The try catch is a stop-gap fix so that the pipeline doesn't stop here
        #When it fails in that manner the file is downloaded and can be used
        wave_watch_files = DataFileManager.get_latest_wave_watch_files()
    except Exception:
        print '-' * 60
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    sst_files = DataFileManager.fetch_new_files()   # not calling as a task so it runs inline
    wind_files = DataFileManager.get_wind_file()

    # If no new files were returned, don't plot or tile anything.
    try:
        #This try catch is also for the wave watch timeout bug
        if not wave_watch_files and not sst_files and not wind_files:
            print "No New Files Available, Quitting."
            return None
    except Exception:
        print '-' * 60
        traceback.print_exc(file=sys.stdout)
        print '-' * 60

    # get the list of plotting tasks based on the files we just downloaded.
    plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days()

    list_of_chains = []

    for pt in plot_task_list:
        if pt.args[0] != 4 and pt.args[0] != 6 and pt.args[0] != 7:
            # chaining passes the result of first function to second function
            list_of_chains.append(chain(pt, tile_overlay.s()))
        else:
            #Use the Wavewatch tiler for Wavewatch files
            list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))

    job = group(item for item in list_of_chains)
    print "jobs:"
    for each in job:
        print each
    #and run the group.
    result = job.apply_async()
    return result
예제 #2
0
def home(request):
    #Models determines which models are displayed on the website. They will appear in the order provided by models[]. Change this order to change the order of the buttons and which buttons appear.
    # OSU_ROMS_SST = 1
    # OSU_ROMS_SUR_SAL = 2
    # OSU_ROMS_SUR_CUR = 3
    # OSU_WW3_HI = 4
    # NAMS_WIND = 5
    # OSU_WW3_DIR = 6
    # OSU_ROMS_BOT_SAL = 7
    # OSU_ROMS_BOT_TEMP = 8
    # OSU_ROMS_SSH = 9
    # NCEP_WW3_DIR = 10
    # NCEP_WW3_HI = 11
    # HYCOM_SST = 12 # RTOFS - Not Used - needs to be renamed
    # HYCOM_SUR_CUR = 13 # RTOFS - Not Used - needs to be renamed
    # OSU_ROMS_TCLINE = 14
    # OSU_ROMS_PCLINE = 15
    # NAVY_HYCOM_SST = 16
    # NAVY_HYCOM_SUR_CUR = 17
    # NAVY_HYCOM_SUR_SAL = 18
    # NAVY_HYCOM_SSH = 19
    # NAVY_HYCOM_BOT_TEMP = 20
    # NAVY_HYCOM_BOT_CUR = 21
    # NAVY_HYCOM_BOT_SAL = 22

    models = [settings.OSU_ROMS_SST,
              settings.OSU_ROMS_SUR_CUR,
              settings.NCEP_WW3_HI,
              settings.NCEP_WW3_DIR,
              settings.NAMS_WIND,
              settings.OSU_ROMS_BOT_TEMP,
              settings.OSU_ROMS_SUR_SAL,
              settings.OSU_ROMS_BOT_SAL,
              settings.OSU_ROMS_SSH,
              settings.OSU_ROMS_TCLINE,
              ]

    extended_models = [settings.OSU_ROMS_SST,
                       settings.OSU_ROMS_SUR_CUR]

    fields = get_list_of_overlay_definitions(models)

    base_overlays = OverlayManager.get_next_few_days_of_tiled_overlays(models)

    if settings.EXTEND:
        extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts('NCDF')
        overlays = base_overlays | extended_overlays # Union of all three querysets - '|' represents the union
    else:
        overlays = base_overlays

    datetimes = overlays.values_list('applies_at_datetime', flat=True).distinct().order_by('applies_at_datetime')
    context = {'overlays': overlays, 'defs': fields, 'times':datetimes }

    return render(request, 'index.html', context)
예제 #3
0
파일: manage.py 프로젝트: MiCurry/SharkEyes
def test(ids=None, navy=False, ncep=False):
    from pl_download.models import DataFile
    from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
    from pl_plot.models import OverlayManager as om
    print ids


    if navy:
        print "NAVY HYCOM TEST"
        count = 0
        if ids:
            for id in ids:
                datafile = DataFile.objects.get(pk=id)
                if datafile.type != 'HYCOM':
                    continue # Skip over non HYCOM Files

                count += 1

            print "HYCOM TESTING ", count, " FOR FILE: "
            info(datafile)
            print "\tTESTING PLOTTER:"
            plotter = NavyPlotter(datafile.file.name)
            if plotter:
                print "\t PLOTTER LOADED SUCCESFULLY"
            else:
                print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name

            print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
            print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()

            print "\t GENERATING PLOT.... "
            print ""

            if datafile.file.name.endswith("ssh.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
            if datafile.file.name.endswith("temp_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
            if datafile.file.name.endswith("temp_bot.nc"):
                om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
            if datafile.file.name.endswith("cur_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
            if datafile.file.name.endswith("sal_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)

    if ncep:
        from pl_download.models import DataFileManager as dm

        print dm.ww3_download_openDAP()

    print "TESTING TASK CREATION"
예제 #4
0
파일: manage.py 프로젝트: wsf1990/SharkEyes
def test(ids=None, navy=False, ncep=False):
    from pl_download.models import DataFile
    from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
    from pl_plot.models import OverlayManager as om
    print ids

    if navy:
        print "NAVY HYCOM TEST"
        count = 0
        if ids:
            for id in ids:
                datafile = DataFile.objects.get(pk=id)
                if datafile.type != 'HYCOM':
                    continue  # Skip over non HYCOM Files

                count += 1

            print "HYCOM TESTING ", count, " FOR FILE: "
            info(datafile)
            print "\tTESTING PLOTTER:"
            plotter = NavyPlotter(datafile.file.name)
            if plotter:
                print "\t PLOTTER LOADED SUCCESFULLY"
            else:
                print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name

            print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times(
            )
            print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()

            print "\t GENERATING PLOT.... "
            print ""

            if datafile.file.name.endswith("ssh.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
            if datafile.file.name.endswith("temp_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
            if datafile.file.name.endswith("temp_bot.nc"):
                om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
            if datafile.file.name.endswith("cur_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
            if datafile.file.name.endswith("sal_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)

    if ncep:
        from pl_download.models import DataFileManager as dm

        print dm.ww3_download_openDAP()

    print "TESTING TASK CREATION"
예제 #5
0
파일: views.py 프로젝트: avaleske/SharkEyes
def home(request):
    #overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays().exclude(definition_id=5)
    overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays()
    #Wind overlay is different from the main overlay due to the time intervals
    #wind_overlays_view_data = overlays_view_data.filter(definition_id=5)

    datetimes = [ i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p') for i in overlays_view_data ]

    #winddatetimes = [ i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p') for i in wind_overlays_view_data ]


    # a complete hack! it just divides a list of all of the times for all the overlays by the number
    # of defs to get a singular list of overlay times

  #  num_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="Wind"))

   # num_wind_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="SST").exclude(display_name_short="Currents"))

    # Team 2 says: at this time we want SST (1), currents (3), and Wave (4) only. Modify this as
    # you add new models.
    num_defs = len(OverlayDefinition.objects.filter(is_base=True, id__in=[1,3,4]))


    list_of_times = datetimes[:len(datetimes)/num_defs]

    #list_of_wind_times = winddatetimes[:len(winddatetimes)/num_wind_defs]


    #context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True).exclude(id=4), 'times':list_of_times, 'windoverlays': wind_overlays_view_data, 'winddefs': OverlayDefinition.objects.filter(id=5), 'windtimes':list_of_wind_times}


    context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True, id__in=[1,3,4]), 'times':list_of_times }

    return render(request, 'index.html', context)
예제 #6
0
파일: views.py 프로젝트: MiCurry/SharkEyes
def testplot(request):
    #Validate User Permissions
    if request.user.is_anonymous():
        return HttpResponse("You do not have admin permissions to this site. Please contact your page administrator.")
    else:
        group_results = OverlayManager.make_all_base_plots_for_next_few_days()
        name_list = [result.get() for result in group_results]
        return HttpResponse("plotted files " + name_list.__str__())
예제 #7
0
파일: manage.py 프로젝트: wsf1990/SharkEyes
def extend():
    from pl_plot.models import OverlayManager

    models = [
        settings.OSU_ROMS_SST,  # Extended
        settings.OSU_ROMS_SUR_SAL,
        settings.OSU_ROMS_SUR_CUR,  # Extended
        settings.OSU_WW3_HI,  # Extended
        settings.NAMS_WIND,
        settings.OSU_WW3_DIR,  # Extended
        settings.OSU_ROMS_BOT_SAL,
        settings.OSU_ROMS_BOT_TEMP,
        settings.OSU_ROMS_SSH,
        settings.OSU_ROMS_TCLINE,
    ]

    extended_models = [
        settings.OSU_ROMS_SST, settings.OSU_ROMS_SUR_CUR, settings.OSU_WW3_HI,
        settings.OSU_WW3_DIR
    ]

    # 14 = Thermocline
    #models = [1,3,4,6,5,8,2,7,9,]
    fields = get_list_of_overlay_definitions(models)

    base_overlays = OverlayManager.get_next_few_days_of_tiled_overlays(models)

    ww3_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts(
        'WAVE', extended_models)
    roms_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts(
        'NCDF', extended_models)

    print "Base Overlays:"
    for overlay in base_overlays:
        print overlay.file, overlay.applies_at_datetime, overlay.is_extend

    print "Extended WW3"
    for overlay in ww3_extended_overlays:
        print overlay.file, overlay.applies_at_datetime, overlay.is_extend

    print "Extended ROMS"
    for overlay in roms_extended_overlays:
        print overlay.file, overlay.applies_at_datetime, overlay.is_extend

    overlays = base_overlays | ww3_extended_overlays | roms_extended_overlays  # Union of all three querysets - '|' represents the union
예제 #8
0
파일: views.py 프로젝트: seacast/SharkEyes
def home(request):
    #This determines which models are ran
    #1 = SST, 3 = Currents, 4 = Wave Height, 5 = Winds, 6 = Wave Direction
    #-------------------------------------------------------------------------
    models = [1,3,4,5,6]
    overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays(models)
    datetimes = overlays_view_data.values_list('applies_at_datetime', flat=True).distinct().order_by('applies_at_datetime')
    context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True, id__in=models), 'times':datetimes }
    return render(request, 'index.html', context)
예제 #9
0
파일: views.py 프로젝트: wsf1990/SharkEyes
def testplot(request):
    #Validate User Permissions
    if request.user.is_anonymous():
        return HttpResponse(
            "You do not have admin permissions to this site. Please contact your page administrator."
        )
    else:
        group_results = OverlayManager.make_all_base_plots_for_next_few_days()
        name_list = [result.get() for result in group_results]
        return HttpResponse("plotted files " + name_list.__str__())
예제 #10
0
파일: manage.py 프로젝트: MiCurry/SharkEyes
def extend():
    from pl_plot.models import OverlayManager

    models = [settings.OSU_ROMS_SST, # Extended
              settings.OSU_ROMS_SUR_SAL,
              settings.OSU_ROMS_SUR_CUR, # Extended
              settings.OSU_WW3_HI, # Extended
              settings.NAMS_WIND,
              settings.OSU_WW3_DIR, # Extended
              settings.OSU_ROMS_BOT_SAL,
              settings.OSU_ROMS_BOT_TEMP,
              settings.OSU_ROMS_SSH,
              settings.OSU_ROMS_TCLINE,
              ]

    extended_models = [settings.OSU_ROMS_SST,
                       settings.OSU_ROMS_SUR_CUR,
                       settings.OSU_WW3_HI,
                       settings.OSU_WW3_DIR]

    # 14 = Thermocline
    #models = [1,3,4,6,5,8,2,7,9,]
    fields = get_list_of_overlay_definitions(models)

    base_overlays = OverlayManager.get_next_few_days_of_tiled_overlays(models)

    ww3_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts('WAVE', extended_models)
    roms_extended_overlays = OverlayManager.get_next_few_days_of_tiled_overlays_for_extended_forecasts('NCDF', extended_models)

    print "Base Overlays:"
    for overlay in base_overlays:
        print overlay.file, overlay.applies_at_datetime, overlay.is_extend

    print "Extended WW3"
    for overlay in ww3_extended_overlays:
        print overlay.file, overlay.applies_at_datetime, overlay.is_extend

    print "Extended ROMS"
    for overlay in roms_extended_overlays:
        print overlay.file, overlay.applies_at_datetime, overlay.is_extend

    overlays = base_overlays | ww3_extended_overlays | roms_extended_overlays # Union of all three querysets - '|' represents the union
예제 #11
0
def do_pipeline():
    print "TASKS: before deleting files"

    DataFileManager.delete_old_files()
    print "TASKS: after deleting Datafiles"
    OverlayManager.delete_old_files()
    print "TASKS: after deleting Overlays"

    wave_watch_files = DataFileManager.get_latest_wave_watch_files()
    print "TASKS: after getting wave files"

    other_files = DataFileManager.fetch_new_files(
    )  # not calling as a task so it runs inline
    print "TASKS: after getting sst/currents files"

    # If no new files were returned, don't plot or tile anything.
    if not wave_watch_files and not other_files:
        return None

    # get the list of plotting tasks based on the files we just downloaded.
    plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days(
    )

    list_of_chains = []

    for pt in plot_task_list:

        if pt.args[0] != 4:
            # chaining passes the result of first function to second function
            list_of_chains.append(chain(pt, tile_overlay.s()))

        else:
            #Use the Wavewatch tiler for Wavewatch files
            list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))

    job = group(item for item in list_of_chains)
    print "jobs:"
    for each in job:
        print each
    #and run the group.
    result = job.apply_async()
    return result
예제 #12
0
def home(request):
    #overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays().exclude(definition_id=5)
    overlays_view_data = OverlayManager.get_next_few_days_of_tiled_overlays()
    #Wind overlay is different from the main overlay due to the time intervals
    #wind_overlays_view_data = overlays_view_data.filter(definition_id=5)

    datetimes = [
        i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p')
        for i in overlays_view_data
    ]

    #winddatetimes = [ i.applies_at_datetime.astimezone(tz.tzlocal()).strftime('%D, %I %p') for i in wind_overlays_view_data ]

    # a complete hack! it just divides a list of all of the times for all the overlays by the number
    # of defs to get a singular list of overlay times

    #  num_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="Wind"))

    # num_wind_defs = len(OverlayDefinition.objects.filter(is_base=True).exclude(display_name_short="SST").exclude(display_name_short="Currents"))

    # Team 2 says: at this time we want SST (1), currents (3), and Wave (4) only. Modify this as
    # you add new models.
    num_defs = len(
        OverlayDefinition.objects.filter(is_base=True, id__in=[1, 3, 4]))

    list_of_times = datetimes[:len(datetimes) / num_defs]

    #list_of_wind_times = winddatetimes[:len(winddatetimes)/num_wind_defs]

    #context = {'overlays': overlays_view_data, 'defs': OverlayDefinition.objects.filter(is_base=True).exclude(id=4), 'times':list_of_times, 'windoverlays': wind_overlays_view_data, 'winddefs': OverlayDefinition.objects.filter(id=5), 'windtimes':list_of_wind_times}

    context = {
        'overlays': overlays_view_data,
        'defs': OverlayDefinition.objects.filter(is_base=True,
                                                 id__in=[1, 3, 4]),
        'times': list_of_times
    }

    return render(request, 'index.html', context)
예제 #13
0
파일: manage.py 프로젝트: wsf1990/SharkEyes
def plot(ids=[],
         num_plots=DEF_NUM_PLOTS,
         tile_flag=DEF_TILE_FLAG,
         full_roms=DEF_FULL_ROMS_FLAG,
         roms=False,
         wave=False,
         wind=False,
         hycom=False,
         ncep=False,
         tcline=False,
         navy=False):
    '''  Just generates plots. You need to pass in the df id to get a plot! Pass it in manually
    or by using one of the functions below which grabs them using the database or via downloading!
    '''
    print "IDS", ids

    if not ids:
        print "PLOT: NO IDS SUBMITTED TO BE PLOTTED - exiting"
        return

    if len(ids) == 0:
        print "PLOT: Empty List of IDS exiting"
        return

    from pl_plot.models import OverlayManager as om
    from pl_chop.tasks import tile_overlay

    if roms:  # OSU ROMS
        roms = []

        print "PLOT: Plotting Roms with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                print "PLOT: OSU ROMS SST - timeslice: ", i
                roms.append(om.make_plot(settings.OSU_ROMS_SST, i, id))
                print "PLOT: OSU ROMS SSC - timeslice: ", i
                roms.append(om.make_plot(settings.OSU_ROMS_SUR_CUR, i, id))

                if full_roms:
                    print "PLOT: Plotting full roms"
                    print "PLOT: OSU ROMS SSC - timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_SUR_SAL, i, id))
                    print "PLOT: OSU ROMS BOT Sal- timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_BOT_SAL, i, id))
                    print "PLOT: OSU ROMS BOT Temp- timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_BOT_TEMP, i,
                                             id))
                    print "PLOT: OSU ROMS SSH - timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_SSH, i, id))

        if tile_flag:
            print "PLOT: Tiling ROMS"
            tile(roms)

        return

    if wave:  # OSU WAVE WATCH III - NO LONGER AVIABLE
        waves = []

        print ids
        print "PLOT: Plotting OSU WW3 with file IDS: ", ids

        if not ids:
            print "Empty List of IDS for OSU WW3"
            return

        for id in ids:
            for i in range(num_plots):
                waves.append(
                    om.make_wave_watch_plot(settings.OSU_WW3_HI, i, id))
                waves.append(
                    om.make_wave_watch_plot(settings.OSU_WW3_DIR, i, id))

        if tile_flag:
            print "PLOT: Tiling waves"
            tile(waves)

        return

    if wind:  # NORTH AMERICAN MESOSCALE - SURFACE WINDS
        winds = []

        print "PLOT: Plotting NAM Winds with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                winds.append(om.make_plot(settings.NAMS_WIND, i, id))

        if tile_flag:
            print "PLOT: Tiling NAM Winds"
            tile(winds)

        return

    if hycom:  # NOAA HYCOM - Not currently implemented on the seacast.org
        hycoms = []

        print "PLOT: Plotting HYCOM with file IDS: ", ids
        for id in ids:
            hycoms.append(om.make_plot(settings.HYCOM_SST, 0, id))
            hycoms.append(om.make_plot(settings.HYCOM_SUR_CUR, 0, id))

        if tile_flag:
            print "PLOT: Tiling HYCOM"
            tile(hycoms)

        return

    if ncep:  # NCEP WAVE WATCH III
        nceps = []

        print ids
        print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                nceps.append(
                    om.make_wave_watch_plot(settings.NCEP_WW3_DIR, i, id))
                nceps.append(
                    om.make_wave_watch_plot(settings.NCEP_WW3_HI, i, id))

        if tile_flag:
            print "PLOT: Tiling NCEP"
            tile(nceps)

        return

    if tcline:  # OSU ROMS THERMOCLINE
        tcline_ids = []

        print ids
        print "PLOT: Plotting TCLINE with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                tcline_ids.append(om.make_plot(settings.OSU_ROMS_TCLINE, i,
                                               id))

        if tile_flag:
            print "PLOT: Tiling tcline"
            tile(tcline_ids)

        return

    if navy:  # NAVY HYCOM
        navy_ids = []

        print ids
        print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                navy_ids.append(om.make_plot(settings.NAVY_HYCOM_SST, i, id))
                navy_ids.append(
                    om.make_plot(settings.NAVY_HYCOM_SUR_CUR, i, id))

        if tile_flag:
            print "PLOT: Tiling NCEP"
            tile(navy_ids)

        return
예제 #14
0
 def get_task_to_tile_next_few_days_of_untiled_overlays(cls):
     overlay_ids_to_tile = OverlayManager.get_next_few_days_of_untiled_overlay_ids(
     )
     return cls.get_tiling_task_by_ids(overlay_ids_to_tile)
예제 #15
0
파일: manage.py 프로젝트: seacast/SharkEyes
 if sys.argv[-1] == "plot":
     from pl_download.models import DataFileManager, DataFile
     from pl_plot.models import OverlayManager
     from pl_chop.tasks import tile_overlay, tile_wave_watch_overlay
     wave = 0
     sst = 0
     wind = 1
     if wave:
         wave = DataFileManager.get_latest_wave_watch_files()
         wave = DataFile.objects.filter(type='WAVE').latest('model_date')
         tiles = []
         begin = time.time()
         #first entry is day-1 at 12pm
         #need to offset 16 to match with sst plot
         #NOTE it increments in 1 hour changes
         tiles += OverlayManager.make_wave_watch_plot(4, 16, wave[0])
         tiles += OverlayManager.make_wave_watch_plot(6, 16, wave[0])
         for t in tiles:
             tile_wave_watch_overlay(t)
         finish = time.time()
         totalTime = (finish - begin)/ 60
         print "Time taken for Waves = " + str(round(totalTime, 2)) + " minutes"
     if sst:
         sst = DataFileManager.fetch_new_files()
         tiles = []
         #first entry is day at 4am
         #NOTE it increments in 4 hour changes
         begin = time.time()
         tiles += OverlayManager.make_plot(1, 0, sst[0])
         tiles += OverlayManager.make_plot(3, 0, sst[0])
         for t in tiles:
예제 #16
0
파일: manage.py 프로젝트: MiCurry/SharkEyes
def plot(ids=[],
         num_plots=DEF_NUM_PLOTS, tile_flag=DEF_TILE_FLAG, full_roms=DEF_FULL_ROMS_FLAG,
         roms=False, wave=False, wind=False, hycom=False, ncep=False, tcline=False, navy=False):
    '''  Just generates plots. You need to pass in the df id to get a plot! Pass it in manually
    or by using one of the functions below which grabs them using the database or via downloading!
    '''
    print "IDS", ids

    if not ids:
        print "PLOT: NO IDS SUBMITTED TO BE PLOTTED - exiting"
        return

    if len(ids) == 0:
        print "PLOT: Empty List of IDS exiting"
        return

    from pl_plot.models import OverlayManager as om
    from pl_chop.tasks import tile_overlay

    if roms: # OSU ROMS
        roms = []

        print "PLOT: Plotting Roms with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                print "PLOT: OSU ROMS SST - timeslice: ", i
                roms.append(om.make_plot(settings.OSU_ROMS_SST, i, id))
                print "PLOT: OSU ROMS SSC - timeslice: ", i
                roms.append(om.make_plot(settings.OSU_ROMS_SUR_CUR, i, id))

                if full_roms:
                    print "PLOT: Plotting full roms"
                    print "PLOT: OSU ROMS SSC - timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_SUR_SAL, i, id))
                    print "PLOT: OSU ROMS BOT Sal- timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_BOT_SAL, i, id))
                    print "PLOT: OSU ROMS BOT Temp- timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_BOT_TEMP, i, id))
                    print "PLOT: OSU ROMS SSH - timeslice: ", i
                    roms.append(om.make_plot(settings.OSU_ROMS_SSH, i, id))

        if tile_flag:
            print "PLOT: Tiling ROMS"
            tile(roms)

        return

    if wave: # OSU WAVE WATCH III - NO LONGER AVIABLE
        waves = []

        print ids
        print "PLOT: Plotting OSU WW3 with file IDS: ", ids

        if not ids:
            print "Empty List of IDS for OSU WW3"
            return

        for id in ids:
            for i in range(num_plots):
                waves.append(om.make_wave_watch_plot(settings.OSU_WW3_HI, i, id))
                waves.append(om.make_wave_watch_plot(settings.OSU_WW3_DIR, i, id))

        if tile_flag:
            print "PLOT: Tiling waves"
            tile(waves)

        return

    if wind: # NORTH AMERICAN MESOSCALE - SURFACE WINDS
        winds = []

        print "PLOT: Plotting NAM Winds with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                winds.append(om.make_plot(settings.NAMS_WIND, i, id))

        if tile_flag:
            print "PLOT: Tiling NAM Winds"
            tile(winds)

        return

    if hycom: # NOAA HYCOM - Not currently implemented on the seacast.org
        hycoms = []

        print "PLOT: Plotting HYCOM with file IDS: ", ids
        for id in ids:
            hycoms.append(om.make_plot(settings.HYCOM_SST, 0, id))
            hycoms.append(om.make_plot(settings.HYCOM_SUR_CUR, 0, id))

        if tile_flag:
            print "PLOT: Tiling HYCOM"
            tile(hycoms)

        return

    if ncep: # NCEP WAVE WATCH III
        nceps = []

        print ids
        print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                nceps.append(om.make_wave_watch_plot(settings.NCEP_WW3_DIR, i, id))
                nceps.append(om.make_wave_watch_plot(settings.NCEP_WW3_HI, i, id))

        if tile_flag:
            print "PLOT: Tiling NCEP"
            tile(nceps)

        return

    if tcline: # OSU ROMS THERMOCLINE
        tcline_ids = []

        print ids
        print "PLOT: Plotting TCLINE with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                tcline_ids.append(om.make_plot(settings.OSU_ROMS_TCLINE, i, id))

        if tile_flag:
            print "PLOT: Tiling tcline"
            tile(tcline_ids)

        return

    if navy: # NAVY HYCOM
        navy_ids = []

        print ids
        print "PLOT: Plotting NCEP WW3 with file IDS: ", ids
        for id in ids:
            for i in range(num_plots):
                navy_ids.append(om.make_plot(settings.NAVY_HYCOM_SST, i, id))
                navy_ids.append(om.make_plot(settings.NAVY_HYCOM_SUR_CUR, i, id))


        if tile_flag:
            print "PLOT: Tiling NCEP"
            tile(navy_ids)

        return
예제 #17
0
파일: manage.py 프로젝트: MiCurry/manage.py
def test(ids=None, navy=False, ncep=False):
    from pl_download.models import DataFile
    from pl_plot.plotter import NavyPlotter, NcepWW3Plotter
    from pl_plot.models import OverlayManager as om
    print ids


    if navy:
        print "NAVY HYCOM TEST"
        count = 0
        if ids:
            for id in ids:
                datafile = DataFile.objects.get(pk=id)
                if datafile.type != 'HYCOM':
                    continue # Skip over non HYCOM Files

                count += 1

            print "HYCOM TESTING ", count, " FOR FILE: "
            info(datafile)
            print "\tTESTING PLOTTER:"
            plotter = NavyPlotter(datafile.file.name)
            if plotter:
                print "\t PLOTTER LOADED SUCCESFULLY"
            else:
                print "\t ERROR: UNABLE TO LOAD FILE: ", datafile.file.name

            print "\t NUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
            print "\t OCEAN TIME: ", plotter.get_time_at_oceantime_index()

            print "\t GENERATING PLOT.... "
            print ""

            if datafile.file.name.endswith("ssh.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
            if datafile.file.name.endswith("temp_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SST, 0, id)
            if datafile.file.name.endswith("temp_bot.nc"):
                om.make_plot(settings.NAVY_HYCOM_BOT_TEMP, 0, id)
            if datafile.file.name.endswith("cur_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_CUR, 0, id)
            if datafile.file.name.endswith("sal_top.nc"):
                om.make_plot(settings.NAVY_HYCOM_SUR_SAL, 0, id)

    if ncep:
        print "NCEP TEST"
        count = 0
        for id in ids:
            datafile = DataFile.objects.get(pk=id)
            if datafile.type != 'NCEP':
                continue # Skip over non HYCOM Files

            count += 1

            print "NCEP WW3 TESTING ", count, " FOR FILE: "
            info(datafile)

            print "   TESTING PLOTTER:"
            plotter = NcepWW3Plotter(datafile.file.name)
            if plotter:
                print "   PLOTTER LOADED SUCCESFULLY"
            else:
                print "   ERROR: UNABLE TO LOAD FILE: ", datafile.file.name

            print "\tNUMBER OF MODEL TIMES: ", plotter.get_number_of_model_times()
            print "\tOCEAN TIMES & TASKS: "
            list = om.get_tasks_for_base_plots_for_next_few_days()
            for i in range(plotter.get_number_of_model_times()):
                print "\t", list[i], "-- time:", plotter.get_oceantime(i)

            print "   GENERATING PLOTS.... "
            cnt = 0
            for i in range(plotter.get_number_of_model_times()):
                print " PLOT: ", cnt, " of", plotter.get_number_of_model_times()
                om.make_wave_watch_plot(settings.NCEP_WW3_DIR, i, id)
                cnt += 1
                print " PLOT: ", cnt, " of", plotter.get_number_of_model_times()
                om.make_wave_watch_plot(settings.NCEP_WW3_HI, i, id)
                cnt += 1

    print "TESTING TASK CREATION"
예제 #18
0
파일: tasks.py 프로젝트: MiCurry/SharkEyes
def do_pipeline():

    logging.info('DO_PIPELINE STARTED')


    # Cleaning up old files from the database and the disk
    print "CLEANING UP - DELETING OLD FILES"
    logging.info('CLEANING UP - DELETING OLD DATAFILE FILES')
    try:
        DataFileManager.delete_old_files()
    except Exception:
        print '-' * 60
        print "COULD NOT DELETE OLD NETCDF FILES"
        logging.error('CLEANING UP - ERROR DELETING DATAFILES')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('CLEANING UP - OLD DATAFILE FILES DELETED')


    logging.info('CLEANING UP - DELETING OLD OVERLAY FILES')
    try:
        OverlayManager.delete_old_files()
    except Exception:
        print '-' * 60
        print "COULD NOT DELETE OVERLAY FILES"
        logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED')


    # Check for new feedback surveys or comments, and email them to Flaxen
    print "SENDING OUT FEEDBACK"
    logging.info('SENDING OUT FEEDBACK')
    try:
        FeedbackHistory.send_feedback_forms()
    except Exception:
        print '-' * 60
        print "COULD NOT SEND FEEDBACK"
        logging.error('ERROR SENDING FEEDBACK FORMS')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('FEEDBACK FORMS SENT')


    logging.info('SENDING OUT SURVEY')
    try:
        FeedbackQuestionaire.send_feedback_survey()
    except Exception:
        print '-' * 60
        print "COULD NOT SEND FEEDBACK SURVEY"
        logging.error('ERROR SENDING FEEDBACK SURVEY')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('SURVEYS Sent')


    print "DOWNLOADING OSU ROMS FILES"
    logging.info('DOWNLOADING OSU ROMS')
    try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
        sst_files = DataFileManager.download_osu_roms()
    except Exception:
        print '-' * 60
        print "COULD NOT DOWNLOAD OSU ROMS FILES"
        logging.error('ERROR DOWNLOADING OSU ROMS')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')

    if settings.WW3_OPENDAP is True:
        print "DOWNLOADING NCEP FILES VIA OPENDAP"
        logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP')
        try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
            wave_watch_files = DataFileManager.ww3_download_openDAP()
        except Exception:
            print '-' * 60
            print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP"
            logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP')
            traceback.print_exc(file=sys.stdout)
            print '-' * 60
        logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY')
    else:
        print "DOWNLOADING UCAR NCEP FILES"
        logging.info('DOWNLOADING UCAR NCEP WW3')
        try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
            wave_watch_files = DataFileManager.ww3_download()
        except Exception:
            print '-' * 60
            print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES"
            logging.error('ERROR DOWNLOADING UCAR NCEP WW3')
            traceback.print_exc(file=sys.stdout)
            print '-' * 60
        logging.info('OSU WW3 DOWNLOADED SUCCESFULLY')


    if settings.EXTEND:
        print "DOWNLOADING NAVY HYCOM"
        logging.info('DOWNLOADING NAVY HYCOM')
        try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
            hycom_files = DataFileManager.navy_hycom_download()
        except Exception:
            print '-' * 60
            print "COULD NOT DOWNLOAD NAVY HYCOM FILES"
            logging.error('ERROR DOWNLOADING OSU ROMS')
            traceback.print_exc(file=sys.stdout)
            print '-' * 60
        logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')


    print "DOWNLOADING WIND FILES"
    logging.info('DOWNLOADING NAM WIND')
    try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
        wind_files = DataFileManager.get_wind_file()
    except Exception:
        print '-' * 60
        print "COULD NOT DOWNLOAD WIND FILE"
        logging.error('ERROR DOWNLOADING NAM WINDS')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('NAM WINDS DOWNLOADED SUCCESFULLY')


    print "DOWNLOADING TCLINE FILES"
    try: # Try Catches to ensure do_pipeline completes even if a model server cant be reached
        tcline_files = DataFileManager.download_tcline()
    except Exception:
        print '-' * 60
        print "COULD NOT DOWNLOAD TCLINE FILE"
        traceback.print_exc(file=sys.stdout)
        print '-' * 60


    try:
        #This try catch is also for the wave watch timeout bug
        if not wave_watch_files and not sst_files and not wind_files \
            and not hycom_files and not tcline_files:
            print "No New Files Available, Quitting."
            return None
    except Exception:
        print '-' * 60
        traceback.print_exc(file=sys.stdout)
        print '-' * 60


    # Get the list of plotting tasks based on the files we just downloaded.
    logging.info('GENERATING TASK LIST')
    plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days()

    list_of_chains = []

    for pt in plot_task_list:
        if pt.args[0] != 4 and pt.args[0] != 6:
            # Chaining passes the result of first function to second function
            list_of_chains.append(chain(pt, tile_overlay.s()))
        else:
            # Use the Wavewatch tiler for Wavewatch files
            list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
    logging.info('TASK LIST GENERATED')

    job = group(item for item in list_of_chains)

    print "PIPELINE: JOBS: "
    for each in job:
        print each

    logging.info('APPLY JOBS')
    result = job.apply_async() # Run the group.
    logging.info('JOBS APPLIED')
    return result
예제 #19
0
파일: models.py 프로젝트: MiCurry/SharkEyes
 def get_task_to_tile_next_few_days_of_untiled_overlays(cls):
     overlay_ids_to_tile = OverlayManager.get_next_few_days_of_untiled_overlay_ids()
     return cls.get_tiling_task_by_ids(overlay_ids_to_tile)
예제 #20
0
def do_pipeline():

    logging.info('DO_PIPELINE STARTED')

    # Cleaning up old files from the database and the disk
    print "CLEANING UP - DELETING OLD FILES"
    logging.info('CLEANING UP - DELETING OLD DATAFILE FILES')
    try:
        DataFileManager.delete_old_files()
    except Exception:
        print '-' * 60
        print "COULD NOT DELETE OLD NETCDF FILES"
        logging.error('CLEANING UP - ERROR DELETING DATAFILES')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('CLEANING UP - OLD DATAFILE FILES DELETED')

    logging.info('CLEANING UP - DELETING OLD OVERLAY FILES')
    try:
        OverlayManager.delete_old_files()
    except Exception:
        print '-' * 60
        print "COULD NOT DELETE OVERLAY FILES"
        logging.error('CLEANING UP - ERROR DELETING OVERLAYS OR TILES')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('CLEANING UP - OLD OVERLAY and TILES DELETED')

    # Check for new feedback surveys or comments, and email them to Flaxen
    print "SENDING OUT FEEDBACK"
    logging.info('SENDING OUT FEEDBACK')
    try:
        FeedbackHistory.send_feedback_forms()
    except Exception:
        print '-' * 60
        print "COULD NOT SEND FEEDBACK"
        logging.error('ERROR SENDING FEEDBACK FORMS')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('FEEDBACK FORMS SENT')

    logging.info('SENDING OUT SURVEY')
    try:
        FeedbackQuestionaire.send_feedback_survey()
    except Exception:
        print '-' * 60
        print "COULD NOT SEND FEEDBACK SURVEY"
        logging.error('ERROR SENDING FEEDBACK SURVEY')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('SURVEYS Sent')

    print "DOWNLOADING OSU ROMS FILES"
    logging.info('DOWNLOADING OSU ROMS')
    try:  # Try Catches to ensure do_pipeline completes even if a model server cant be reached
        sst_files = DataFileManager.download_osu_roms()
    except Exception:
        print '-' * 60
        print "COULD NOT DOWNLOAD OSU ROMS FILES"
        logging.error('ERROR DOWNLOADING OSU ROMS')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')

    if settings.WW3_OPENDAP is True:
        print "DOWNLOADING NCEP FILES VIA OPENDAP"
        logging.info('DOWNLOADING NCEP WW3 VIA OPENDAP')
        try:  # Try Catches to ensure do_pipeline completes even if a model server cant be reached
            wave_watch_files = DataFileManager.ww3_download_openDAP()
        except Exception:
            print '-' * 60
            print "COULD NOT DOWNLOAD NCEP WW3 FILE VIA OPENDAP"
            logging.error('ERROR DOWNLOADING NCEP WW3 OPENDAP')
            traceback.print_exc(file=sys.stdout)
            print '-' * 60
        logging.info('NCEP WW3 VIA OPENDAP DOWNLOADED SUCCESFULLY')
    else:
        print "DOWNLOADING UCAR NCEP FILES"
        logging.info('DOWNLOADING UCAR NCEP WW3')
        try:  # Try Catches to ensure do_pipeline completes even if a model server cant be reached
            wave_watch_files = DataFileManager.ww3_download()
        except Exception:
            print '-' * 60
            print "COULD NOT DOWNLOAD UCAR NCEP WW3 FILES"
            logging.error('ERROR DOWNLOADING UCAR NCEP WW3')
            traceback.print_exc(file=sys.stdout)
            print '-' * 60
        logging.info('OSU WW3 DOWNLOADED SUCCESFULLY')

    if settings.EXTEND:
        print "DOWNLOADING NAVY HYCOM"
        logging.info('DOWNLOADING NAVY HYCOM')
        try:  # Try Catches to ensure do_pipeline completes even if a model server cant be reached
            hycom_files = DataFileManager.navy_hycom_download()
        except Exception:
            print '-' * 60
            print "COULD NOT DOWNLOAD NAVY HYCOM FILES"
            logging.error('ERROR DOWNLOADING OSU ROMS')
            traceback.print_exc(file=sys.stdout)
            print '-' * 60
        logging.info('OSU ROMS DOWNLOADED SUCCESFULLY')

    print "DOWNLOADING WIND FILES"
    logging.info('DOWNLOADING NAM WIND')
    try:  # Try Catches to ensure do_pipeline completes even if a model server cant be reached
        wind_files = DataFileManager.get_wind_file()
    except Exception:
        print '-' * 60
        print "COULD NOT DOWNLOAD WIND FILE"
        logging.error('ERROR DOWNLOADING NAM WINDS')
        traceback.print_exc(file=sys.stdout)
        print '-' * 60
    logging.info('NAM WINDS DOWNLOADED SUCCESFULLY')

    print "DOWNLOADING TCLINE FILES"
    try:  # Try Catches to ensure do_pipeline completes even if a model server cant be reached
        tcline_files = DataFileManager.download_tcline()
    except Exception:
        print '-' * 60
        print "COULD NOT DOWNLOAD TCLINE FILE"
        traceback.print_exc(file=sys.stdout)
        print '-' * 60

    try:
        #This try catch is also for the wave watch timeout bug
        if not wave_watch_files and not sst_files and not wind_files \
            and not hycom_files and not tcline_files:
            print "No New Files Available, Quitting."
            return None
    except Exception:
        print '-' * 60
        traceback.print_exc(file=sys.stdout)
        print '-' * 60

    # Get the list of plotting tasks based on the files we just downloaded.
    logging.info('GENERATING TASK LIST')
    plot_task_list = OverlayManager.get_tasks_for_base_plots_for_next_few_days(
    )

    list_of_chains = []

    for pt in plot_task_list:
        if pt.args[0] != 4 and pt.args[0] != 6:
            # Chaining passes the result of first function to second function
            list_of_chains.append(chain(pt, tile_overlay.s()))
        else:
            # Use the Wavewatch tiler for Wavewatch files
            list_of_chains.append(chain(pt, tile_wave_watch_overlay.s()))
    logging.info('TASK LIST GENERATED')

    job = group(item for item in list_of_chains)

    print "PIPELINE: JOBS: "
    for each in job:
        print each

    logging.info('APPLY JOBS')
    result = job.apply_async()  # Run the group.
    logging.info('JOBS APPLIED')
    return result
예제 #21
0
파일: manage.py 프로젝트: MiCurry/manage.py
def plot_by_id(ids=None):
    if ids == None:
        print "PLOT_BY_ID: NO IDS SUPPLIED ... Quitting"
        return 0

    from pl_download.models import DataFile as df
    from pl_plot.models import OverlayManager as om

    for id in ids:
        file = df.objects.get(pk=id)
        print_file_info(file)

        if file.type == 'NCDF':
            om.make_plot()

        elif file.type == 'WAVE':
            om.make_plot()

        elif file.type == 'WIND':
            om.make_plot()

        elif file.type == 'T-CLINE':
            om.make_plot()

        elif file.type == 'NCEP_WW3':
            om.make_plot()

        elif file.type == 'HYCOM':
            om.make_plot()

        elif file.type == 'RTOFS':
            om.make_plot()