def getNaturalEarthDsets(conf): """retrieves the available natural earth datasets""" reponame = "nvkelso/natural-earth-vector" commitsha = "bf7720b54dd9ac2d4d7f735174901b3862b5362a" cachedir = conf.getCacheDir("githubcache") try: cred = conf.authCred("github", ['oauthtoken']) token = cred.oauthtoken except: token = None # import pdb;pdb.set_trace() catalog = cachedGithubCatalogue(reponame, cachedir=cachedir, commitsha=commitsha, gfilter=ghfilter({ "type": "blob", "path": "\.geojson" }), gfollowfilter=ghfilter({ "type": "tree", "path": "geojson" }), ghtoken=token) out = [] #create a list of datasets for entry in catalog["datasets"]: clsname = os.path.basename(entry["path"]).split(".")[0] out.append(NaturalEarthClassFactory(clsname, entry)) return out
def getGeoshapesDsets(conf): """retrieves the available geoshapes datasets""" currentversion = 'v1' # note this correspond to a specific release (the sha hash is taken from the commit) reponame = "strawpants/geoshapes" commitsha = "23ab6daab0bf2c6e9a3f3d685bed52cb48ac97ea" cachedir = conf.getCacheDir("githubcache") try: cred = conf.authCred("github", ['oauthtoken']) token = cred.oauthtoken except: token = None # import pdb;pdb.set_trace() catalog = cachedGithubCatalogue(reponame, cachedir=cachedir, commitsha=commitsha, gfilter=ghfilter({ "type": "blob", "path": "\.geojson" }), gfollowfilter=ghfilter({ "type": "tree", "path": "envelopes" }), ghtoken=token) out = [] #create a list of datasets for entry in catalog["datasets"]: clsname = os.path.basename(entry["path"]).split(".")[0] out.append(GeoshapeClassFactory(clsname, entry)) return out
def pull(self): """Pulls the dataset from github and unpacks it in the cache directory""" #crawls the github repository for Load Love numbers reponame = "strawpants/snrei" commitsha = "e61d3e2a9eb328d48f56f5aa73fa2aaba60f1d5c" try: cred = self.conf.authCred("github", ['oauthtoken']) token = cred.oauthtoken except: token = None # import pdb;pdb.set_trace() ghcrawler = ghCrawler(reponame, commitsha=commitsha, filter=ghfilter({ "type": "blob", "path": "\.love" }), followfilt=ghfilter({ "type": "tree", "path": "Love" }), oauthtoken=token) #download all datasets ghcrawler.parallelDownload(self.dataDir(), check=True, maxconn=3, gzip=True)