Esempio n. 1
0
def extractSFOModelData(lat_lon_list, name_list):
    print "Extract SFOModelData from http://opendap.co-ops.nos.noaa.gov/thredds/ "
    urls = buildSFOUrls(jd_start, jd_stop)
    index = -1
    #setup struct
    df_list = {}
    for n in name_list:
        df_list[n] = {}

    data_dates = []
    for i, url in enumerate(urls):
        try:
            #print url
            nc = netCDF4.Dataset(url, 'r')
        except:
            #print  "\tNot Available"
            break

        if i == 0:
            lats = nc.variables['lat'][:]
            lons = nc.variables['lon'][:]
            lons = lons - 360
            index_list, dist_list = findSFOIndexs(lats, lons, lat_lon_list)

        #Extract the model data using and MF dataset
        time_dim = nc.variables['time']
        u_dim = nc.variables['u']
        v_dim = nc.variables['v']

        u_var = u_dim[:, 0, index_list]
        v_var = v_dim[:, 0, index_list]

        #create the dates
        dates = num2date(time_dim[:],
                         units=time_dim.units,
                         calendar='gregorian')
        #data_dates.append(dates)

        for i, n in enumerate(name_list):
            #get lat and lon
            df_list[n]['lat'] = lats[index_list[i]]
            df_list[n]['lon'] = lons[index_list[i]]
            #create speed and direction, convert
            ws = uv2ws(u_var[:, i] * 100, v_var[:, i] * 100)
            wd = uv2wd(u_var[:, i] * 100, v_var[:, i] * 100)

            data = {}
            data['sea_water_speed (cm/s)'] = ws
            data['direction_of_sea_water_velocity (degree)'] = wd
            columns = [
                'sea_water_speed (cm/s)',
                'direction_of_sea_water_velocity (degree)'
            ]

            df = pd.DataFrame(data=data, index=dates, columns=columns)
            #create struct
            if 'data' in df_list[n]:
                df_list[n]['data'] = df_list[n]['data'].append(df)
            else:
                df_list[n]['data'] = df

    return df_list
Esempio n. 2
0
def get_hr_radar_dap_data(dap_urls, st_list, jd_start, jd_stop):
    # Use only data within 1.00 degrees
    obs_df = []
    obs_or_model = False
    max_dist = 1.0
    # Use only data where the standard deviation of the time series exceeds 0.01 m (1 cm).
    # This eliminates flat line model time series that come from land points that should have had missing values.
    min_var = 0.1
    data_idx = []

    df_list = []
    for url in dap_urls:
        #only look at 6km hf radar
        if 'http://hfrnet.ucsd.edu/thredds/dodsC/HFRNet/USWC/' in url and "6km" in url and "GNOME" in url:
            print url
            #get url
            nc = netCDF4.Dataset(url, 'r')
            lat_dim = nc.variables['lat']
            lon_dim = nc.variables['lon']
            time_dim = nc.variables['time']
            u_var = None
            v_var = None
            for key in nc.variables.iterkeys():
                key_dim = nc.variables[key]
                try:
                    if key_dim.standard_name == "surface_eastward_sea_water_velocity":
                        u_var = key_dim
                    elif key_dim.standard_name == "surface_northward_sea_water_velocity":
                        v_var = key_dim
                    elif key_dim.standard_name == "time":
                        time = key_dim
                except:
                    #only if the standard name is not available
                    pass

            #manage dates
            dates = num2date(time_dim[:],
                             units=time_dim.units,
                             calendar='gregorian')
            date_idx = []
            date_list = []
            for i, date in enumerate(dates):
                if jd_start < date < jd_stop:
                    date_idx.append(i)
                    date_list.append(date)
            #manage location
            for st in st_list:
                station = st_list[st]
                f_lat = station['lat']
                f_lon = station['lon']

                ret = find_nearest(f_lat, f_lon, lat_dim[:], lon_dim[:])
                lat_idx = ret[0]
                lon_idx = ret[1]
                dist_deg = ret[2]
                #print "lat,lon,dist=",ret

                if len(u_var.dimensions) == 3:
                    #3dimensions
                    ret = cycleAndGetData(u_var, v_var, date_idx, lat_idx,
                                          lon_idx)
                    u_vals = ret[0]
                    v_vals = ret[1]

                    lat_idx = ret[2]
                    lon_idx = ret[3]

                    print "lat,lon,dist=", ret[2], ret[3]
                try:
                    #turn vectors in the speed and direction
                    ws = uv2ws(u_vals, v_vals)
                    wd = uv2wd(u_vals, v_vals)

                    data_spd = []
                    data_dir = []
                    data = {}
                    data['sea_water_speed (cm/s)'] = np.array(ws)
                    data[
                        'direction_of_sea_water_velocity (degree)'] = np.array(
                            wd)
                    time = np.array(date_list)

                    df = pd.DataFrame(
                        data=data,
                        index=time,
                        columns=[
                            'sea_water_speed (cm/s)',
                            'direction_of_sea_water_velocity (degree)'
                        ])
                    df_list.append({
                        "name": st,
                        "data": df,
                        "lat": lat_dim[lat_idx],
                        "lon": lon_dim[lon_idx],
                        "ws_pts": np.count_nonzero(~np.isnan(ws)),
                        "wd_pts": np.count_nonzero(~np.isnan(wd)),
                        "dist": dist_deg,
                        'from': url
                    })
                except Exception, e:
                    print "\t\terror:", e
        else:
            pass