def pull(self,
name=None,
wsne=None,
tstart=datetime.min,
tend=datetime.max):
"""Pulls a subset of the imerg gridded dataset as netcdf from an motu enabled server
This routine calls the internal routines of the motuclient python client
:param name: Name of the output datatset (file will be named 'name.nc')
:param wsne: bounding box of the section of interest as [West,South,North,East]
:param tstart: start date (as yyyy-mm-dd) for the extraction
:param tend: end date (as yyyy-mm-dd) for the extraction
"""
if name is not None:
self.queueDownloadjob(name, wsne, tstart, tend)
auth = self.conf.authCred("podaac")
ddir = self.dataDir()
for ky, val in self.pullqueue.items():
url = self.rooturl + val
fout = os.path.join(ddir, ky + ".nc")
http(url,
cookiefile=self.cookiefile,
auth=auth,
lastmod=datetime(2022, 7, 1)).download(ddir,
outfile=fout,
check=True)
def pullRGI(downloaddir,comparewithversion):
httpserv=http('http://www.glims.org/RGI/rgi60_files/00_rgi60.zip',lastmod=datetime(2018,1,1))
#Newest version which is supported by this plugin
newestver=(6,0)
upd=False
#now determine whether to retrieve the file
if newestver > comparewithversion:
uri,upd=httpserv.download(downloaddir,check=True)
if not os.path.exists(os.path.join(downloaddir,'extract')):
#unzip all the goodies
zipd=os.path.join(downloaddir,'zipfiles')
with ZipFile(uri.url,'r') as zp:
zp.extractall(zipd)
#now recursively zip the other zip files
for zf in glob(zipd+'/*zip'):
slurplogger().info("Unzipping %s"%(zf))
with ZipFile(zf,'r') as zp:
zp.extractall(os.path.join(downloaddir,'extract'))
#remove zipfiles after extracting
shutil.rmtree(zipd)
#Patches one csv file which contains a . instead of a , at one point
#download patch from github
pf='04_rgi60_ArcticCanadaSouth_hypso.csv.patch'
slurplogger().info("Patching csv file %s"%(pf) )
httpget=http("https://raw.githubusercontent.com/strawpants/geoslurp/master/patches/"+pf)
uri,pupd=httpget.download(os.path.join(downloaddir,'extract'),check=True)
#apply patch
subprocess.Popen(['patch','-i',pf],cwd=os.path.dirname(uri.url))
else:
slurplogger().info("RGI 6.0 already downloaded")
return newestver,upd
def pull(self):
"""Download ascii file"""
httpserv = http(os.path.join(
'https://icesat4.gsfc.nasa.gov/cryo_data/drainage_divides/',
self.fbase),
lastmod=datetime(2020, 1, 10))
uri, upd = httpserv.download(self.cacheDir(), check=True, gzip=True)
def pullFoG(downloaddir):
"""Pulls various GRDC datasets from the webdav folder"""
fogSource=http("http://www.wgms.ch/downloads/DOI-WGMS-FoG-2018-06.zip")
urif,upd=fogSource.download(downloaddir)
if upd:
with ZipFile(urif.url,'r') as zp:
zp.extractall(downloaddir)
def download(self):
if self.uri.startswith('http'):
uri=http(self.uri,lastmod=self.meta['lastupdate'])
elif self.uri.startswith('ftp'):
uri=ftp(self.uri)
uri.updateModTime()
uri.download(self.direc,check=True,outfile=self.fout)
def pullHydro(hytype, downloaddir):
# see https://www.dropbox.com/sh/hmpwobbz9qixxpe/AAAI_jasMJPZl_6wX6d3vEOla for the 'root' of the hydrsheds data
hysource = {
"hybas_af":
"https://www.dropbox.com/sh/hmpwobbz9qixxpe/AADoPLdVZNd2JG-KaJNY0zT1a/HydroBASINS/standard/af/hybas_af_lev01-06_v1c.zip",
"hybas_eu":
"https://www.dropbox.com/sh/hmpwobbz9qixxpe/AABz1Pym5esD6GUJcnzaaqpEa/HydroBASINS/standard/eu/hybas_eu_lev01-06_v1c.zip",
"af_riv_30s":
"https://www.dropbox.com/sh/hmpwobbz9qixxpe/AAC9imuUajl_1bS0tKWqPE8Ya/HydroSHEDS_RIV/RIV_30s/af_riv_30s.zip",
"eu_riv_30s":
"https://www.dropbox.com/sh/hmpwobbz9qixxpe/AAD68vqkhRNJd5qK3NVvM7TSa/HydroSHEDS_RIV/RIV_30s/eu_riv_30s.zip"
}
httpserv = http(hysource[hytype], lastmod=datetime(2021, 2, 8))
#Newest version which is supported by this plugin
uri, upd = httpserv.download(downloaddir, check=True)
if upd:
#unzip all the goodies
zipd = os.path.join(downloaddir, 'extract')
with ZipFile(uri.url, 'r') as zp:
zp.extractall(zipd)
else:
slurplogger().info(
"This component of hydrosheds is already downloaded")
return upd
def pullFoG(downloaddir):
"""Pulls a zip archive of the WGMS data"""
fogSource = http("http://www.wgms.ch/downloads/DOI-WGMS-FoG-2018-06.zip")
urif, upd = fogSource.download(downloaddir)
if upd:
with ZipFile(urif.url, 'r') as zp:
zp.extractall(downloaddir)
def getSubTree(self, url):
if self.token:
#add the api token to the header
headers = [f"Authorization: token {self.token}"]
else:
headers = None
return json.loads(http(url, headers=headers).buffer().getvalue())
def pull(self, intersect=None):
# download the entire mosaic domain in one tif
if self.res in ['1km', '500m', '100m']:
rasteruri = http(
"http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/"
+ self.res + "/" + self.rasterfile,
lastmod=datetime(2018, 9, 26))
rasterfileuri, upd = rasteruri.download(self.srcdir, check=False)
def pull(self):
"""Pulls the google kml files from the copernicus server"""
rooturl = 'https://sentinel.esa.int/documents/247904/685098/Sentinel-3-Absolute-Ground-Tracks.zip'
cache = self.cacheDir()
httpserv = http(rooturl, lastmod=datetime(2021, 11, 29))
uri, upd = httpserv.download(cache, check=True)
if upd:
with ZipFile(uri.url, 'r') as zp:
zp.extractall(cache)
def pull(self):
"""Pulls the dataset from github and unpacks it in the cache directory"""
#download the inventory file
lastchanged = datetime(2021, 11, 5)
inventory = "https://github.com/strawpants/GRACE-filter/raw/master/inventory.xlsx"
uri, upd = http(inventory,
lastmod=lastchanged).download(self.cacheDir(),
check=True)
pdinvent = pd.read_excel(uri.url, engine="openpyxl")
#download all the files
ddir = self.dataDir()
for idx, row in pdinvent.iterrows():
ffile, upd = http(row.uri,
lastmod=lastchanged).download(ddir, check=True)
#update file with newly downloaded file
pdinvent.at[idx, 'uri'] = self.conf.generalize_path(ffile.url)
#write updated excel file
pdinvent.to_csv(os.path.join(self.pdfile))
def loadCoordinates(self):
"""Download the time, lon and lat coordinates and load these coordinates from the netdf file"""
if self.dscoords is not None:
return
auth = self.conf.authCred("podaac")
cache = self.cacheDir()
coordfile = os.path.join(cache, "Coordinates.nc")
opendapcoordinates = self.rooturl + "?time[0:1:255],lat[0:1:1799],lon[0:1:3599]"
http(opendapcoordinates,
cookiefile=self.cookiefile,
auth=auth,
lastmod=datetime(2022, 7,
1)).download(cache,
outfile=os.path.basename(coordfile),
check=True)
self.dscoords = xr.open_dataset(coordfile, decode_times=True)
def pull(self):
"""Download acsii files in zip and unpack ascii data"""
httpserv = http(
'https://github.com/AustralianAntarcticDivision/orsifronts/raw/master/data-raw/fronts.zip',
lastmod=datetime(2018, 1, 1))
uri, upd = httpserv.download(self.cacheDir(), check=True)
if upd:
#unpack zip
with ZipFile(uri.url, 'r') as zp:
zp.extractall(self.cacheDir())
def pull(self):
"""Pulls the shapefile from the aviso server"""
httpserv = http(os.path.join(
'https://www.aviso.altimetry.fr/fileadmin/documents/missions/Swot/',
self.zipf),
lastmod=datetime(2019, 9, 6))
cache = self.cacheDir()
uri, upd = httpserv.download(cache, check=True)
if upd:
with ZipFile(os.path.join(cache, self.zipf), 'r') as zp:
zp.extractall(cache)
def pull(self):
"""Pulls the shapefile layers from the server"""
zipf = http(
"http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/indexes/" +
self.filebase + ".zip",
lastmod=datetime(2018, 9, 26))
#download the zip shapefiles
downloaddir = self.cacheDir()
uri, upd = zipf.download(downloaddir, check=True)
zipd = os.path.join(downloaddir, 'extract')
if not os.path.exists(zipd):
#unzip the goodies
with ZipFile(uri.url, 'r') as zp:
slurplogger().info("Unzipping %s" % (uri.url))
zp.extractall(zipd)
def pull(self):
"""Pulls the wribasin data from the internet and unpacks it in the cache directory"""
#wrisource=http("http://www.fao.org/geonetwork/srv/en/resources.get")
wrisource = http(
"http://www.fao.org/geonetwork/srv/en/resources.get?id=30914&fname=wri_basins.zip&access=private"
)
# urif, upd=wrisource.download(self.cacheDir(),check=True,outfile=os.path.join(self.cacheDir(),"wri_basin.zip"),postdic={"id":"30914","fname":"wri_basins.zip","access":"private"})
urif, upd = wrisource.download(self.cacheDir(),
check=True,
outfile=os.path.join(
self.cacheDir(), "wri_basin.zip"))
if upd:
with ZipFile(urif.url, 'r') as zp:
zp.extractall(self.cacheDir())
def pull(self):
"""Pulls the geocenter ascii files in the cache"""
uri=http("https://wobbly.earth/data/"+self.fout,lastmod=datetime(2018,10,16)).download(self.cacheDir(),check=True)
def pull(self):
"""Pulls the geojson data from github and unpacks it in the cache directory"""
uri = http(self.url)
basename = os.path.basename(self.path)
uri.download(direc=self.cacheDir(), outfile=basename)
def pull(self):
"""Pulls the geocenter ascii files in the cache"""
uri=http("https://podaac-tools.jpl.nasa.gov/drive/files/allData/grace/docs/"+self.fout,lastmod=datetime(2019,12,1)).download(self.cacheDir(),check=True)
def pull(self):
"""Pulls the geocenter ascii files in the cache"""
uri=http("http://download.csr.utexas.edu/pub/slr/geocenter/"+self.fout30).download(self.cacheDir())
uri=http("http://download.csr.utexas.edu/pub/slr/geocenter/"+self.fout60).download(self.cacheDir())
def getCatalog(url, auth=None):
"""Retrieve a catalogue"""
slurplogger().info("getting Thredds catalog: %s" % (url))
buf = http(url, auth=auth).buffer()
return XMLTree.fromstring(buf.getvalue())
def pull(self):
"""Pulls the OBP matlab file from the cloud"""
cred=self.conf.authCred("awipies",['url','user','passw'])
obpsource=http(cred.url,auth=cred)
obpsource.download(self.dataDir(),outfile=self.obpfile)
def __init__(self):
super().__init__(url="http://icgem.gfz-potsdam.de/tom_longtime")
buf=http(self.rooturl).buffer()
self._roothtml=HTMLtree(buf.getvalue())
def getSubTree(self, url):
if self.token:
#add the api token to the end
url += "?access_token=%s" % (self.token)
return json.loads(http(url).buffer().getvalue())
def uris(self, depth=10):
"""Construct Uris from tree nodes"""
for item in self.treeitems(depth=depth):
yield http(item["url"])
def pull(self):
http(self.url).download(self.cacheDir())
zpf=os.path.join(self.cacheDir(),os.path.basename(self.url))
#unzip file
with ZipFile(zpf,'r') as zp:
zp.extractall(self.cacheDir())
def pull(self):
upsrc = http(
"https://confluence.ecmwf.int/download/attachments/143039724/upArea.nc",
lastmod=datetime(2021, 11, 17))
#download to cache only (will be in db raster)
urif, upd = upsrc.download(self.srcdir, check=True)
