def getNaturalEarthDsets(conf):
"""retrieves the available natural earth datasets"""
reponame = "nvkelso/natural-earth-vector"
commitsha = "bf7720b54dd9ac2d4d7f735174901b3862b5362a"
cachedir = conf.getCacheDir("githubcache")
try:
cred = conf.authCred("github", ['oauthtoken'])
token = cred.oauthtoken
except:
token = None
# import pdb;pdb.set_trace()
catalog = cachedGithubCatalogue(reponame,
cachedir=cachedir,
commitsha=commitsha,
gfilter=ghfilter({
"type": "blob",
"path": "\.geojson"
}),
gfollowfilter=ghfilter({
"type": "tree",
"path": "geojson"
}),
ghtoken=token)
out = []
#create a list of datasets
for entry in catalog["datasets"]:
clsname = os.path.basename(entry["path"]).split(".")[0]
out.append(NaturalEarthClassFactory(clsname, entry))
return out
def getGeoshapesDsets(conf):
"""retrieves the available geoshapes datasets"""
currentversion = 'v1' # note this correspond to a specific release (the sha hash is taken from the commit)
reponame = "strawpants/geoshapes"
commitsha = "23ab6daab0bf2c6e9a3f3d685bed52cb48ac97ea"
cachedir = conf.getCacheDir("githubcache")
try:
cred = conf.authCred("github", ['oauthtoken'])
token = cred.oauthtoken
except:
token = None
# import pdb;pdb.set_trace()
catalog = cachedGithubCatalogue(reponame,
cachedir=cachedir,
commitsha=commitsha,
gfilter=ghfilter({
"type": "blob",
"path": "\.geojson"
}),
gfollowfilter=ghfilter({
"type": "tree",
"path": "envelopes"
}),
ghtoken=token)
out = []
#create a list of datasets
for entry in catalog["datasets"]:
clsname = os.path.basename(entry["path"]).split(".")[0]
out.append(GeoshapeClassFactory(clsname, entry))
return out
def pull(self):
"""Pulls the dataset from github and unpacks it in the cache directory"""
#crawls the github repository for Load Love numbers
reponame = "strawpants/snrei"
commitsha = "e61d3e2a9eb328d48f56f5aa73fa2aaba60f1d5c"
try:
cred = self.conf.authCred("github", ['oauthtoken'])
token = cred.oauthtoken
except:
token = None
# import pdb;pdb.set_trace()
ghcrawler = ghCrawler(reponame,
commitsha=commitsha,
filter=ghfilter({
"type": "blob",
"path": "\.love"
}),
followfilt=ghfilter({
"type": "tree",
"path": "Love"
}),
oauthtoken=token)
#download all datasets
ghcrawler.parallelDownload(self.dataDir(),
check=True,
maxconn=3,
gzip=True)