def windspeed(self): varu="u-component_of_wind_surface" varv="v-component_of_wind_surface" uVel=[] vVel=[] for month_year_str in self.__years_months: month,year=month_year_str.split("-") days=monthrange(int(year),int(month))[1] base_url=self.__urlfunc("wind",month,year) if(isinstance(self.__lon1,int)): url=base_url+varu+self.__varstr(days)+","+varv+self.__varstr(days) print(url) request=open_dods(url) #print(month,year,time.time()-t) #u_vel=request[varu][varu]__calc_maxt_idx__calc_maxt_idx__calc_maxt_idx__calc_maxt_idx #v_vel=request[varv][varv] u_vel=request[varu] v_vel=request[varv] uVel.append(u_vel) vVel.append(v_vel) else: url1=base_url+varu+self.__varstr[0](days)+","+varv+self.__varstr[0](days) url2=base_url+varu+self.__varstr[1](days)+","+varv+self.__varstr[1](days) request1=open_dods(url1) request2=open_dods(url2) u_vel1=request1[varu] v_vel1=request1[varv] u_vel2=request2[varu] v_vlatsrangeel2=request2[varv] uVel.append((u_vel1,u_vel2)) vVel.append((v_vel1,v_vel2)) return uVel,vVel
def waveheight(self): var_height="Significant_height_of_combined_wind_waves_and_swell_surface" waveheights=[] for month_year_str in self.__years_months: month,year=month_year_str.split("-") days=monthrange(int(year),int(month))[1] base_url=self.__urlfunc("hs",month,year) if(isinstance(self.__lon1,int)): url=base_url+var_height+self.__varstr(days) #print(url) print(month_year_str,url) request=open_dods(url) wave_height=request[var_height] waveheights.append(wave_height) else: url1=base_url+var_height+self.__varstr[0](days) url2=base_url+var_height+self.__varstr[1](days) #print(url1,url2) request1=open_dods(url1) request2=open_dods(url2) wave_height1=request1[var_height] wave_height2=request2[var_height] waveheights.append((wave_height1,wave_height2)) return waveheights
def __init__(self,): self.__base_url="https://tds.hycom.org/thredds/dodsC/GLBv0.08/expt_93.0/uv3z.dods?" self.__start_date="2000-01-01 00:00:00" self.__mindepth=self.__maxdepth=0 self.__minlat=-80 self.__minlon=0 self.__maxlon=359.92 url=self.__base_url+"time" laturl=self.__base_url+"lat" print(url) self._timesmap=np.asarray(open_dods(url)["time"]) self._latsmap=np.around(np.asarray(open_dods(laturl)["lat"]),2)
def test_timeout(sequence_type_data): """Test that timeout works properly""" TestDataset = DatasetType('Test') TestDataset['sequence'] = sequence_type_data TestDataset['byte'] = BaseType('byte', 0) application = BaseHandler(TestDataset) # Explictly add latency on the devel server # to guarantee that it timeouts def wrap_mocker(func): def mock_add_latency(*args, **kwargs): time.sleep(1e-1) return func(*args, **kwargs) return mock_add_latency application = wrap_mocker(application) with LocalTestServer(application) as server: url = ("http://0.0.0.0:%s/" % server.port) # test open_url assert open_url(url) == TestDataset with pytest.raises(HTTPError) as e: open_url(url, timeout=1e-5) assert 'Timeout' in str(e) # test open_dods with pytest.raises(HTTPError): open_dods(url + '.dods?sequence', timeout=1e-5) assert 'Timeout' in str(e) # test sequenceproxy dataset = open_url(url) seq = dataset['sequence'] assert isinstance(seq.data, SequenceProxy) # Change the timeout of the sequence proxy: seq.data.timeout = 1e-5 with pytest.raises(HTTPError) as e: next(seq.iterdata()) assert 'Timeout' in str(e) # test baseproxy: dat = dataset['byte'] assert isinstance(dat.data, BaseProxy) # Change the timeout of the baseprox proxy: dat.data.timeout = 1e-5 with pytest.raises(HTTPError) as e: dat[:] assert 'Timeout' in str(e)
def testOpenDap(self): print 'testOpenDap NASA EARTHDATA' # Get authentication from the .netrc file netrc_file = '/home/jli/.netrc' if os.path.isfile(netrc_file): logins = netrc.netrc() accounts = logins.hosts for host, info in accounts.iteritems(): self.login, self.account, self.password = info #dataset_url = 'https://goldsmr5.gesdisc.eosdis.nasa.gov:443/opendap/MERRA2/M2I3NPASM.5.12.4/1986/12/MERRA2_100.inst3_3d_asm_Np.19861201.nc4' dataset_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/dods/M2T1NXSLV.dods?t2m[1:2][1:5][7:14]' dataset_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/dods/M2T1NXSLV.dods?t2m' session = setup_session(self.login, self.password, check_url=dataset_url) #dataset = open_url(dataset_url,session=session) #dataset_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/dods/M2T1NXSLV.dods?t2m[1:2][1:5][7:14]' dataset = open_dods(dataset_url, session=session) #store = xarray.backends.PydapDataStore(dataset) #ds = xarray.open_dataset(store) #print ds.keys() print type(dataset) print dataset.keys()
def getData(self): todayString = self.gmTime.strftime('%Y%m%d') global baseurl oururl = baseurl + todayString + '/nww3' + \ todayString + '_%02dz' % self.tm_hour dodsUrl = oururl + ".dods?" + \ "{2}.{2}[0:1:60][{0}:1:{0}][{1}:1:{1}]".format( self.lattitudeIndex, self.longitudeIndex, self.variable) logging.debug('DODS:' + dodsUrl) try: data = open_dods(dodsUrl) except ServerError: logging.debug('URL DOES NOTEXIST') self.gmTime -= timedelta(hours=6) self.tm_hour = chooseNearestHour(self.gmTime) return self.getData() #logging.debug(data[self.variable][:]) #logging.debug(type(data)) #logging.debug(type(data[self.variable])) for key,value in enumerate(data[self.variable]): if value>9.999e+20: #logging.debug('No data;'+str(key)+':'+str(value)) data[self.variable].data[key]=NODATA return data[self.variable][:]
def getData(self): todayString = self.gmTime.strftime('%Y%m%d') global baseurl oururl = baseurl + todayString + '/nww3' + \ todayString + '_%02dz' % self.tm_hour dodsUrl = oururl + ".dods?" + \ "{2}.{2}[0:1:60][{0}:1:{0}][{1}:1:{1}]".format( self.lattitudeIndex, self.longitudeIndex, self.variable) logging.debug('DODS:' + dodsUrl) try: data = open_dods(dodsUrl) except ServerError: logging.debug('URL DOES NOTEXIST') self.gmTime -= timedelta(hours=6) self.tm_hour = chooseNearestHour(self.gmTime) return self.getData() #logging.debug(data[self.variable][:]) #logging.debug(type(data)) #logging.debug(type(data[self.variable])) for key, value in enumerate(data[self.variable]): if value > 9.999e+20: #logging.debug('No data;'+str(key)+':'+str(value)) data[self.variable].data[key] = NODATA return data[self.variable][:]
def explore(): d = open_url('http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc') ##v = d["SST"] ## ##v["lon"] ## ##v["SST"].shape ## ##s = v["SST"] ## ##print ##(1, 16000, 36000) ##a = d.SST[0,12000:13000,18000:19000] # time:0, lat: 40-50N, lon 0-10E sst = d['SST'] ##grid = sst[0:1:0,0:1:10,0:1:3] ##grid = sst[0:1:0,100:1:110,0:1:3] # generated http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.dods?SST.SST[0:1:0][100:110:0][0:3:0] # which is not correct, it should be: # http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.ascii?SST[0:1:0][0:1:10][0:1:3] ##grid = sst.SST[d.time==0, (d.lat >= 100) & (d.lat <= 110), (d.lon >= 0) & (d.lon <= 3)] from pydap.client import open_dods d2 = open_dods('http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.dods?SST[0:1:0][0:1:10][0:1:3]') ##http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.ascii?SST[0:1:0][1000:1:1010][1000:1:1003] #d = open_url('http://coastwatch.pfeg.noaa.gov/erddap/griddap/erdTAssh1day') ##start = end = datetime(2010, 5, 19, 12, 0, 0) ##ssh = d.ssh.ssh[ ## (d.time >= coards.format(start, d.time.units)) & (d.time <= coards.format(end, d.time.units)), ## (d.altitude == 0), ## (d.latitude >= 17) & (d.latitude <= 32), ## (d.longitude >= 260) & (d.longitude <= 281)] #http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/sst_20130526.nc.ascii?SST[0:1:0][100:1:110][0:1:3] suffix = "2013/147/20130527-JPL_OUROCEAN-L4UHfnd-GLOB-v01-fv01_0-G1SST.nc.bz2" d = open_url("http://podaac-opendap.jpl.nasa.gov/opendap/allData/ghrsst/data/L4/GLOB/JPL_OUROCEAN/G1SST/" + suffix) ##g = d.analysed_sst[0, 12000:12010, 18000:18010] g.lat.data g.lon.data t = g.analysed_sst.data
def test_open_dods_with_attributes(self): """Open the dods response together with the das response.""" dataset = open_dods('.dods', metadata=True, application=self.app) self.assertEqual(dataset.NC_GLOBAL, {}) self.assertEqual(dataset.DODS_EXTRA, {}) self.assertEqual(dataset.description, "A simple sequence for testing.") self.assertEqual(dataset.cast.lon.axis, 'X') self.assertEqual(dataset.cast.lat.axis, 'Y') self.assertEqual(dataset.cast.depth.axis, 'Z') self.assertEqual(dataset.cast.time.axis, 'T') self.assertEqual(dataset.cast.time.units, "days since 1970-01-01")
def test_open_dods_with_attributes(sequence_app): """Open the dods response together with the das response.""" dataset = open_dods('.dods', metadata=True, application=sequence_app) assert (dataset.NC_GLOBAL == {}) assert (dataset.DODS_EXTRA == {}) assert ( dataset.description == "A simple sequence for testing.") assert (dataset.cast.lon.axis == 'X') assert (dataset.cast.lat.axis == 'Y') assert (dataset.cast.depth.axis == 'Z') assert (dataset.cast.time.axis == 'T') assert ( dataset.cast.time.units == "days since 1970-01-01")
def wave_speeds(self,latrange,lonrange,timerange): u_name="water_u" v_name="water_v" minlat,maxlat=self.__map_lats(latrange) minlon,maxlon=self.__map_lons(lonrange) startidx,endidx=self._map_times(timerange) time_string="[{}:1:{}]".format(startidx,endidx) depth_string="[{}:1:{}]".format(self.__mindepth,self.__maxdepth) lat_string="[{}:1:{}]".format(minlat,maxlat) if(isinstance(minlon,int)): lon_string="[{}:1:{}]".format(minlon,maxlon) else: lon_string=("[{}:1:{}]".format(minlon[0],minlon[1]),"[{}:1:{}]".format(maxlon[0],maxlon[1])) if(isinstance(minlon,int)): u_string=u_name+time_string+depth_string+lat_string+lon_string v_string=v_name+time_string+depth_string+lat_string+lon_string url=self.__base_url+u_string+","+v_string req=open_dods(url,timeout=5000) u=req[u_name] v=req[v_name] return u,v else: u_string1=u_name+time_string+depth_string+lat_string+lon_string[0] v_string1=v_name+time_string+depth_string+lat_string+lon_string[0] u_string2=u_name+time_string+depth_string+lat_string+lon_string[1] v_string2=v_name+time_string+depth_string+lat_string+lon_string[1] url1=self.__base_url+u_string1+","+v_string1 url2=self.__base_url+u_string2+","+v_string2 req1=open_dods(url1,timeout=5000) req2=open_dods(url2,timeout=5000) u1=req1[u_name] v1=req1[v_name] u2=req2[u_name] v2=req2[v_name] u=(u1,u2) v=(v1,v2) return u,v
def test_open_dods_with_attributes(self): """Open the dods response together with the das response.""" dataset = open_dods('.dods', metadata=True, application=self.app) self.assertEqual(dataset.NC_GLOBAL, {}) self.assertEqual(dataset.DODS_EXTRA, {}) self.assertEqual( dataset.description, "A simple sequence for testing.") self.assertEqual(dataset.cast.lon.axis, 'X') self.assertEqual(dataset.cast.lat.axis, 'Y') self.assertEqual(dataset.cast.depth.axis, 'Z') self.assertEqual(dataset.cast.time.axis, 'T') self.assertEqual( dataset.cast.time.units, "days since 1970-01-01")
def compute(self): url = self.getInputFromPort("url") dataSlice = self.getInputFromPort("dataSlice") myDataSet = open_dods(str(url) + '.dods?' + str(dataSlice)) result = myDataSet.data #TODO: should return list of arrays but this will require a new type # to be added to eo4vistrails; for instance NDArrayList my_first_data_item_key = myDataSet.keys()[0] dict_of_arrays = myDataSet.get(my_first_data_item_key) for output_port, known_field_name in (('time','time'), ('latitude','latitude'), ('longitude','longitude'), ('data',my_first_data_item_key)): outArray = NDArray() outArray.set_array(dict_of_arrays.get(known_field_name).data) self.setResult(output_port, outArray)
def test_open_dods(self): """Open the dods response from a server. Note that here we cannot simply compare ``dataset.data`` with the original data ``SimpleSequence.data``, because the original data contains int16 values which are transmitted as int32 in the DAP spec. """ dataset = open_dods('.dods', application=self.app) self.assertEqual(list(dataset.data), [[ ('1', 100, -10, 0, -1, 21, 35, 0), ('2', 200, 10, 500, 1, 15, 35, 100), ]]) # attributes should be empty self.assertEqual(dataset.attributes, {})
def test_open_dods_16bits(sequence_app): """Open the dods response from a server. Note that here we cannot simply compare ``dataset.data`` with the original data ``SimpleSequence.data``, because the original data contains int16 values which are transmitted as int32 in the DAP spec. """ dataset = open_dods('.dods', application=sequence_app) assert ( list(dataset.data) == [[ ('1', 100, -10, 0, -1, 21, 35, 0), ('2', 200, 10, 500, 1, 15, 35, 100), ]]) # attributes should be empty assert dataset.attributes == {}
def subset_dataset(self, url, metadata=False): """ Return a subset dataset as Parameters ---------- url : [type] base URL for a dataset that can be opened with pydap.open_url Returns ------- DatasetType : pydap DatasetType """ dods_url = self.create_dods_url(url) logger.debug(f"Downloading subset from {dods_url}") ds = open_dods(dods_url, session=self.session, metadata=metadata) return ds
def wave_speeds2(self,latrange,lonrange,timerange,foldername): df=pd.DataFrame(columns=["date_time","lat","lon","water_u","water_v"]) u_name="water_u" v_name="water_v" minlat,maxlat=self.__map_lats(latrange) minlon,maxlon=self.__map_lons(lonrange) startidx,endidx=self._map_times(timerange) print(startidx,endidx) start_date=datetime.strptime("2000-01-01 00:00:00","%Y-%m-%d %H:%M:%S") depth_string="[{}:1:{}]".format(self.__mindepth,self.__maxdepth) lat_string="[{}:1:{}]".format(minlat,maxlat) if(isinstance(minlon,int)): lon_string="[{}:1:{}]".format(minlon,maxlon) else: lon_string=("[{}:1:{}]".format(minlon[0],minlon[1]),"[{}:1:{}]".format(maxlon[0],maxlon[1])) if(isinstance(minlon,int)): lon_strings=[lon_string] else: lon_strings=[lon_string[0],lon_string[1]] for lonid,lon_string in enumerate(lon_strings): for idx in range(startidx,endidx+1): time_string="[{}:1:{}]".format(idx,idx) if(not os.path.exists(foldername+"/"+str(idx)+"_"+str(lonid)+".npy")): u_string=u_name+time_string+depth_string+lat_string+lon_string v_string=v_name+time_string+depth_string+lat_string+lon_string url=self.__base_url+u_string+","+v_string req=open_dods(url,timeout=50000) u=req[u_name] v=req[v_name] times=int(np.asarray(u["time"])[0]) lats=u["lat"] lons=u["lon"] uv=[u["water_u"][0],v["water_v"][0]] if(idx==startidx): if(lonid==0): np.save(foldername+"/"+"lats",np.asarray(lats)) np.save(foldername+"/"+"lons_{}".format(lonid),np.asarray(lons)) np.save(foldername+"/"+str(idx)+"_"+str(lonid),np.asarray(uv)) return True
def get_sequential(file, time, var_config, lat_idx, lon_idx, verbose=False): var_list = [] nlev_dict = {} for var, config in var_config.items(): if config["type"] == "surface": var_list.append( FORMAT_STR.format(var=var, lat=lat_idx, lon=lon_idx)) nlev_dict[var] = 1 else: lev_idx = tuple(config["levels"]) var_list.append( FORMAT_STR_PL.format(var=var, lev=lev_idx, lat=lat_idx, lon=lon_idx)) nlev_dict[var] = lev_idx[1] - lev_idx[0] + 1 ncoord = (lat_idx[1] - lat_idx[0] + 1) * (lon_idx[1] - lon_idx[0] + 1) request = URL.format(file, time) + ",".join(var_list) if verbose: print(request) try: dataset = open_dods(request) except: raise var_data = [ pd.DataFrame( var.data.reshape(nlev_dict[var.name], ncoord).T, columns=[ "{}{}".format(var.name, n) for n in range(nlev_dict[var.name]) ], ) for var in dataset ] return pd.concat(var_data, axis=1)
def get_file(request, param, var_conf, time, lat, lon, verbose=False): ntime = len(time) ncoord = len(lat) * len(lon) var_list = [ ((FORMAT_STR if vartype == "surface" else FORMAT_STR_PL).format( var=var, **param)) for var, vartype in var_conf.items() ] if verbose: print(request + ",".join(var_list)) try: dataset = open_dods(request + ",".join(var_list)) except: raise OpenFileError("file '{}' not available".format(request[:-1])) var_data = [ var.data.reshape((ntime, -1, ncoord)) for var in dataset.values() ] var_names = [ "{}{}".format(var, n) for idx, var in enumerate(dataset) for n in range(var_data[idx].shape[1]) ] index = pd.MultiIndex.from_product((lat, lon), names=["lat", "lon"]) columns = pd.MultiIndex.from_product((time, var_names), names=["time", "var"]) return pd.DataFrame( (np.concatenate(var_data, axis=1).transpose(2, 0, 1).reshape( ncoord, -1)), index=index, columns=columns, )
#!/usr/bin/env python from pydap.client import open_dods import numpy as np import argparse as ap parser = ap.ArgumentParser(description='Use pydap client to access HDF5 files via Hyrax ') parser.add_argument('url', nargs='?',default='https://eosdap.hdfgroup.org:8080/opendap/data/hdf5/grid_1_2d.h5.dods?temperature[0:1:1][0:1:1]',help='The URL to access the HDF5 files via Hyrax') #Change the URL to your own server's address args=parser.parse_args() input_url =''.join(args.url) #Obtain the data via pydap dataset=open_dods(input_url) data=[] data=dataset.temperature[:] #Generate the output file name from URL last_slash_index=input_url.rfind('/') h5_suffix_index=input_url.find(".h5",last_slash_index) output_string=input_url[last_slash_index+1:h5_suffix_index] #Save the data with numpy savetxt np.savetxt(output_string+'_data.txt',data)
def save_dataset(hour, date, var_config, time_tuple, lat_tuple, lon_tuple, fname, verbose=False): """Download the datasets for a specific date and hour""" date_str = date.strftime("%Y%m%d") month_str = date.strftime("%Y%m") file = DIR.format(month_str, date_str, hour) time_list = list(range1(time_tuple[0], time_tuple[1], 3)) # Get the lat and lon grids from the first dataset present in the server for time in time_list: request = URL.format(file, time) if verbose: print(request) try: coord = open_dods(request + "lat,lon") except: continue else: break try: lat, lon = coord["lat"][:].data, coord["lon"][:].data except: # UnboundLocalError: local variable 'coord' referenced before assignment # none of the 180/3 + 1 datasets where present in the server raise # Transform longitudes from range 0..360 to -180..180 lon = np.where(lon > 180, lon - 360, lon) # Transform into python lists to use the index() method # Actually it would be better to use np.where() lat_list, lon_list = lat.tolist(), lon.tolist() try: lat_idx = (lat_list.index(lat_tuple[1]), lat_list.index(lat_tuple[0])) except: raise ValueError("Latitude not in the grid", lat_tuple) lat = lat[range1(*lat_idx)].tolist() if lon_tuple[0] < 0 and lon_tuple[1] > 0: try: lon_idx_w = (lon_list.index(lon_tuple[0]), len(lon_list) - 1) lon_idx_e = (0, lon_list.index(lon_tuple[1])) except: raise ValueError("Longitude not in the grid", lon_tuple) lon = np.concatenate( (lon[range1(*lon_idx_w)], lon[range1(*lon_idx_e)])).tolist() try: data_list = [ get_general( file, time, var_config, lat_idx, lon_idx_w, lon_idx_e, verbose=verbose, ) for time in time_list ] except: raise else: try: lon_idx = (lon_list.index(lon_tuple[0]), lon_list.index(lon_tuple[1])) except: raise ValueError("Longitude not in the grid", lon_tuple) lon = lon[range1(*lon_idx)].tolist() try: data_list = [ get_sequential(file, time, var_config, lat_idx, lon_idx, verbose=verbose) for time in time_list ] except: raise data = pd.concat(data_list, axis=1, keys=time_list, names=["time", "var"]) data.index = pd.MultiIndex.from_product((lat, lon), names=["lat", "lon"]) data.sort_index(inplace=True) data.to_csv(fname, sep=" ", float_format="%.3f")
def get_general(file, time, var_config, lat_idx, lon_idx_w, lon_idx_e, verbose=False): request = URL.format(file, time) var_w_list, var_e_list = [], [] nlev_dict = {} for var, config in var_config.items(): if config["type"] == "surface": var_w_list.append( FORMAT_STR.format(var=var, lat=lat_idx, lon=lon_idx_w)) var_e_list.append( FORMAT_STR.format(var=var, lat=lat_idx, lon=lon_idx_e)) nlev_dict[var] = 1 else: lev_idx = tuple(config["levels"]) var_w_list.append( FORMAT_STR_PL.format(var=var, lev=lev_idx, lat=lat_idx, lon=lon_idx_w)) var_e_list.append( FORMAT_STR_PL.format(var=var, lev=lev_idx, lat=lat_idx, lon=lon_idx_e)) nlev_dict[var] = lev_idx[1] - lev_idx[0] + 1 ncoord = (lat_idx[1] - lat_idx[0] + 1) * ( (lon_idx_w[1] - lon_idx_w[0] + 1) + (lon_idx_e[1] - lon_idx_e[0] + 1)) request_w = request + ",".join(var_w_list) request_e = request + ",".join(var_e_list) if verbose: print(request_w) print(request_e) try: dataset_w = open_dods(request_w) dataset_e = open_dods(request_e) except: raise # 1. Concatenate both 'west' and 'east' data along 'lon' axis (last axis) # 2. Reshape 4D-array (1, nlev, nlat, nlon) to 2D-array (nlev, nlat*nlon) # 3. Transpose and store in DataFrame # This is ugly :(, partly because the long function names var_data = [ pd.DataFrame( (np.concatenate((var_w.data[0], var_e.data[0]), axis=len(var_w.data[0].shape) - 1).reshape( nlev_dict[var_w.name], ncoord).T), columns=[ "{}{}".format(var_w.name, n) for n in range(nlev_dict[var_w.name]) ], ) for var_w, var_e in zip(dataset_w.values(), dataset_e.values()) ] # 4. Concatenate the DataFrames for each var to obtain a single DataFrame # with (nlat*nlon, nlev*nvar) Note every var can have a different number # of levels (either pressure levels or height above ground levels) return pd.concat(var_data, axis=1)
qpf_datetime_list = tidy_ndfd_var_dates(var_lines_list=qpf_lines) # %% print(pop12_datetime_list) print(qpf_datetime_list) # %% test = min(pop12_datetime_list) test_str = dt.datetime.strftime(test, "%Y%m%d%H%M") test_str # %% #https://www.ncdc.noaa.gov/thredds/catalog/ndfd/file/catalog.html #catalog #temp_data_url = "https://www.ncdc.noaa.gov/thredds/dodsC/ndfd/file/" + "201808/20180822/" + "YDCZ98_KWBN_" + test_str + ".dods" temp_data_url = "https://www.ncdc.noaa.gov/thredds/dodsC/ndfd/file/" + "201808/20180801/" + "YICZ98_KWBN_201808010446.dods" temp_ndfd_data = open_dods(temp_data_url) temp_ndfd_data # %% temp_x = temp_ndfd_data['x'][:] temp_y = temp_ndfd_data['y'][:] #temp_pop12 = temp_ndfd_data['Total_precipitation_surface_12_Hour_Accumulation_probability_above_0p254'] # pop12 temp_qpf = temp_ndfd_data[ 'Total_precipitation_surface_6_Hour_Accumulation'] # qpf # save pop12 dimentions #temp_pop12_dims = temp_pop12.dimensions #temp_pop12_time_dim = temp_pop12_dims[0] # save qpf dimentions temp_qpf_dims = temp_qpf.dimensions
#!/usr/bin/env python from pydap.client import open_dods import numpy as np import argparse as ap parser = ap.ArgumentParser( description='Use pydap client to access HDF5 files via Hyrax ') parser.add_argument( 'url', nargs='?', default= 'https://eosdap.hdfgroup.org:8080/opendap/data/hdf5/grid_1_2d.h5.dods?temperature[0:1:1][0:1:1]', help='The URL to access the HDF5 files via Hyrax') #Change the URL to your own server's address args = parser.parse_args() input_url = ''.join(args.url) #Obtain the data via pydap dataset = open_dods(input_url) data = [] data = dataset.temperature[:] #Generate the output file name from URL last_slash_index = input_url.rfind('/') h5_suffix_index = input_url.find(".h5", last_slash_index) output_string = input_url[last_slash_index + 1:h5_suffix_index] #Save the data with numpy savetxt np.savetxt(output_string + '_data.txt', data)
def save_dataset( fname, date, hour, var_conf, res, step, time_tuple, lev_idx, lat_tuple, lon_tuple, verbose=False, ): request = URL.format( date=date, hour=hour, res="{0:.2f}".format(res).replace(".", "p"), step="" if step == 3 else "_{:1d}hr".format(step), ) if verbose: print(request + "lat,lon") try: coord = open_dods(request + "lat,lon") except: raise OpenFileError("file '{}' not available".format(request[:-1])) # We don't get the time array from the server since it is in seconds from a # date. Instead we compute the times in hours manually. time = range1(*time_tuple, step=step) # TODO: there is a possible problem here if the division is not exact time_idx = (int(time_tuple[0] / step), int(time_tuple[1] / step)) # Slicing [:] downloads the data from the server lat, lon = coord["lat"][:].data, coord["lon"][:].data # Transform longitudes from range 0..360 to -180..180 lon = np.where(lon > 180, lon - 360, lon) # Transform into python lists to use the index() method # TODO: change to find the closest lat/lon with argmin and not an exact match lat_list, lon_list = lat.tolist(), lon.tolist() try: lat_idx = (lat_list.index(lat_tuple[0]), lat_list.index(lat_tuple[1])) except: raise ValueError("Latitude not in the grid", lat_tuple) lat = lat[range1(*lat_idx)].tolist() if lon_tuple[0] < 0 and lon_tuple[1] > 0: try: lon_idx_w = (lon_list.index(lon_tuple[0]), len(lon_list) - 1) lon_idx_e = (0, lon_list.index(lon_tuple[1])) except: raise ValueError("Longitude not in the grid", lon_tuple) lon_w = lon[range1(*lon_idx_w)].tolist() lon_e = lon[range1(*lon_idx_e)].tolist() param_w = { "lat": lat_idx, "lon": lon_idx_w, "time": time_idx, "lev": lev_idx } param_e = { "lat": lat_idx, "lon": lon_idx_e, "time": time_idx, "lev": lev_idx } try: data_w = get_file(request, param_w, var_conf, time, lat, lon_w, verbose=verbose) data_e = get_file(request, param_e, var_conf, time, lat, lon_e, verbose=verbose) except: raise data = pd.concat((data_w, data_e), axis=0) else: try: lon_idx = (lon_list.index(lon_tuple[0]), lon_list.index(lon_tuple[1])) except: raise ValueError("Longitude not in the grid", lon_tuple) lon = lon[range1(*lon_idx)].tolist() param = { "lat": lat_idx, "lon": lon_idx, "time": time_idx, "lev": lev_idx } try: data = get_file(request, param, var_conf, time, lat, lon, verbose=verbose) except: raise data.to_csv(fname, sep=" ", float_format="%.3f")
def test_open_dods_with_attributes(self): dataset = open_dods('http://localhost:8001/.dods', True) self.assertEqual(dataset.attributes['NC_GLOBAL'], {}) self.assertEqual(dataset.attributes['DODS_EXTRA'], {}) self.assertEqual(dataset.attributes['type'], "Drifters")
def test_open_dods(self): dataset = open_dods('http://localhost:8001/.dods') data = np.array(np.rec.fromrecords(DATA, names=dataset.Drifters.keys())) np.testing.assert_array_equal(data, dataset.Drifters.data) self.assertEqual(dataset.attributes, {})
# Example (no data level): # # *NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..* # # Ends up in the following request: # http://dev-ebas-pydap.nilu.no/NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..dods # **We will do a quick example with mercury data from the Zeppelin station** # # We access the data using the OPeNDAP protocol, we send a dods request and recieve binary data. # # This is now parsed by the PyDAP library. # In[4]: mercury_ds = open_dods('http://dev-ebas-pydap.nilu.no/NO0042G.Hg_mon.IMG.air.mercury.1h.NO01L_tekran_42_dup.NO01L_afs..dods') # We get the keys keys = mercury_ds.keys() print(keys) # **Now we can append the data to some variables** # In[5]: mercury = mercury_ds['mercury'] # Let's have a quicklook at the data print(mercury.keys())