furi, upd = geturi.download(self.cache, True) if upd: with ZipFile(furi.url, 'r') as zp: zp.extractall(self.cache) self.updateInvent(False) # Factory method to dynamically create classes def GSHHGClassFactory(clsName): splt = clsName.split("_") if len(splt) == 3: rgx = splt[1] else: rgx = None path = splt[0] + "_shp/" + splt[-1] return type(clsName, (GSHHGBase, ), {"path": path, "layerregex": rgx}) def getGSHHGDsets(conf): """Automatically create all classes contained within the GSHHG database""" out = [] for nm in ["GSHHS", "WDBII_river", "WDBII_border"]: for res in ['c', 'l', 'i', 'h', 'f']: clsName = nm + "_" + res out.append(GSHHGClassFactory(clsName)) return out geoslurpCatalogue.addDatasetFactory(getGSHHGDsets)
mv=[] tv=[] cnmv=[] sigcnmv=[] self.updateInvent() def GRACEGeocFac(conf): out=[] for center in ["CSR", "GFZ", "JPL"]: out.append(type("geocenter_"+center+"RL06_TN13",(geocenter_GRCRL06_TN13,),{"fout":"TN-13_GEOC_"+center+"_RL06.txt"})) return out geoslurpCatalogue.addDatasetFactory(GRACEGeocFac) class geocenter_RIES_CFCM(DataSet): fout30="GCN_L1_L2_30d_CF-CM.txt" fout60="GCN_L1_L2_60d_CF-CM.txt" #note also embed mm to meter conversion in here (e3) sqrt3timesRE=11047256.23312e3 scheme=scheme table=type("geocenter_ries_cfcmTable", (GravitySHTBase,), {}) def __init__(self,dbconn): super().__init__(dbconn) # super().__init__(direc=direc,uri='https://wobbly.earth/data/Geocenter_dec2017.tgz',order=['c10','c11','s11'],lastupdate=datetime(2018,10,16)) def pull(self): """Pulls the geocenter ascii files in the cache"""
def getRGIDsets(conf): """Automatically create all classes contained within the GSHHG database""" out=[] regionnames=['01_rgi60_Alaska', '02_rgi60_WesternCanadaUS', '03_rgi60_ArcticCanadaNorth', '04_rgi60_ArcticCanadaSouth', '05_rgi60_GreenlandPeriphery', '06_rgi60_Iceland', '07_rgi60_Svalbard', '08_rgi60_Scandinavia', '09_rgi60_RussianArctic','10_rgi60_NorthAsia', '11_rgi60_CentralEurope', '12_rgi60_CaucasusMiddleEast', '13_rgi60_CentralAsia', '14_rgi60_SouthAsiaWest', '15_rgi60_SouthAsiaEast', '16_rgi60_LowLatitudes', '17_rgi60_SouthernAndes', '18_rgi60_NewZealand', '19_rgi60_AntarcticSubantarctic'] out.append(RGISHPClassFactory("00_rgi60_O1Regions.shp")) out.append(RGISHPClassFactory("00_rgi60_O2Regions.shp")) out.append(RGICSVClassFactory("00_rgi60_summary.csv", 1)) out.append(RGICSVClassFactory("00_rgi60_links.csv", 2)) for nm in regionnames: out.append(RGISHPClassFactory(nm+".shp")) #also add important csv files for nm in regionnames: out.append(RGICSVClassFactory(nm+"_hypso.csv")) return out geoslurpCatalogue.addDatasetFactory(getRGIDsets)
splt, (SWOTSIMBase, ), { "ogrfile": ogrfile, "zipf": zipfileName, "swapxy": True, "gtype": "GEOMETRY" }) def getSWOTSIMDsets(conf): """Automatically create all classes for the swot simulation tracks""" out = [] swotversions = [ ('swot_calval_orbit_june2015-v2_nadir.shp', 'shp_calval_nadir.zip'), ('swot_calval_orbit_june2015-v2_swath.shp', 'sph_calval_swath.zip'), ('swot_science_orbit_sept2015-v2_10s_nadir.shp', 'swot_science_orbit_sept2015-v2_10s_nadir.zip'), ('swot_science_orbit_sept2015-v2_10s_swath.shp', 'swot_science_orbit_sept2015-v2_10s_swath.zip'), ('swot_science_hr_2.0s_4.0s_June2019-v3_nadir.shp', 'swot_science_hr_2.0s_4.0s_June2019-v3_nadir.zip'), ('swot_science_hr_2.0s_4.0s_June2019-v3_swath.shp', 'swot_science_hr_2.0s_4.0s_June2019-v3_swath.zip') ] for ogr, zipf in swotversions: out.append(SWOTSIMClassFactory(ogr, zipf)) return out geoslurpCatalogue.addDatasetFactory(getSWOTSIMDsets)
# Factory method to dynamically create classes def GRDCGISClassFactory(fileName): splt = fileName.split(".") return type(splt[0], (grdc_gis_base, ), { "filename": fileName, "gtype": "GEOMETRY", "swapxy": True }) def getGRDCDsets(conf): """Automatically create all classes contained within the GRDC tables""" GISshapes = [ 'GRDC_405_basins_from_mouth.shp', 'GRDC_687_rivers.shp', 'GRDC_687_rivers_class.shp', 'GRDC_lakes_join_rivers.shp', 'grdc_basins_smoothed.shp' ] out = [GRDCGISClassFactory(name) for name in GISshapes] #also add the monthly and daily datasets for name in ["grdc_monthly", "grdc_daily"]: out.append(GRDCClassFactory(name)) return out geoslurpCatalogue.addDatasetFactory(getGRDCDsets) geoslurpCatalogue.addDataset(grdc_catalogue)
cachedir = conf.getCacheDir("githubcache") try: cred = conf.authCred("github", ['oauthtoken']) token = cred.oauthtoken except: token = None # import pdb;pdb.set_trace() catalog = cachedGithubCatalogue(reponame, cachedir=cachedir, commitsha=commitsha, gfilter=ghfilter({ "type": "blob", "path": "\.geojson" }), gfollowfilter=ghfilter({ "type": "tree", "path": "envelopes" }), ghtoken=token) out = [] #create a list of datasets for entry in catalog["datasets"]: clsname = os.path.basename(entry["path"]).split(".")[0] out.append(GeoshapeClassFactory(clsname, entry)) return out geoslurpCatalogue.addDatasetFactory(getGeoshapesDsets)
else: files=[UriFile(file) for file in findFiles(self._dbinvent.datadir,'.*gfc.gz',since=self._dbinvent.lastupdate)] newfiles=self.retainnewUris(files) #loop over files for uri in newfiles: slurplogger().info("extracting meta info from %s"%(uri.url)) meta=icgemMetaExtractor(uri) meta=enhanceMeta(meta) self.addEntry(meta) self.updateInvent() def TUGRAZGRACEL2ClassFactory(release,subdirs): """Dynamically construct GRACE Level 2 dataset classes for TU GRAZ""" base,gtype=subdirs.split('/') clsName="_".join([release,gtype]) table=type(clsName.replace('-',"_") +"Table", (GravitySHTBase,), {}) return type(clsName, (TUGRAZGRACEL2Base,), {"release": release, "table":table,"subdirs":subdirs}) # setup GRACE datasets def TUGRAZGRACEDsets(conf): out=[] release='ITSG-Grace2018' for subdirs in ["daily_kalman/daily_background","daily_kalman/daily_n40","monthly/monthly_background","monthly/monthly_n60","monthly/monthly_n96","monthly/monthly_n120"]: out.append(TUGRAZGRACEL2ClassFactory(release,subdirs)) return out geoslurpCatalogue.addDatasetFactory(TUGRAZGRACEDsets)
def hydroriverClassFactory(hytype): fname = "%s.shp" % (hytype) return type(hytype, (HydroshedBase, ), { "hytype": hytype, "filename": fname, "gtype": "GEOMETRY", "swapxy": True }) def getHyRivers(conf): out = [] for hytype in ["af_riv_30s", "eu_riv_30s"]: out.append(hydroriverClassFactory(hytype)) return out def getHyBasins(conf): out = [] for hytype in ["hybas_af", "hybas_eu"]: for lev in range(1, 7): out.append(hydrobasinsClassFactory(hytype, lev)) return out geoslurpCatalogue.addDatasetFactory(getHyRivers) geoslurpCatalogue.addDatasetFactory(getHyBasins)
#open documentation files with open(os.path.join(zipdir,'docu',"%d.txt"%(id))) as docid: data["doc"]=docid.readlines() #open auth file with open(os.path.join(zipdir,'docu',"%d_auth.txt"%(id))) as docid: data["auth"]=docid.readlines() meta['tstart']=tmin meta["tend"]=tmax meta["data"]=data self.addEntry(meta) self.updateInvent() def PSMSLClassFactory(clsName): dum,typ,freq=clsName.lower().split("_") url="http://www.psmsl.org/data/obtaining/"+typ+"."+freq+".data/"+typ+"_"+freq+".zip" table=type(clsName +"Table", (PSMSLTBase,), {}) return type(clsName, (PSMSLBase,), {"url": url, "table":table,"typ":typ,"freq":freq}) def getPSMSLDsets(conf): out=[] for clsName in ["psmsl_rlr_monthly","psmsl_rlr_annual","psmsl_met_monthly"]: out.append(PSMSLClassFactory(clsName)) return out geoslurpCatalogue.addDatasetFactory(getPSMSLDsets)
"A directory/regex with output data needs to be supplied when registering this dataset" ) newfiles = self.retainnewUris( [UriFile(file) for file in findFiles(rundir, pattern)]) for uri in newfiles: meta = FESOMMetaExtractor(uri) if not meta: #don't register empty entries continue self.addEntry(meta) self._dbinvent.data["Description"] = "FESOM output data table" self.setDataDir(os.path.abspath(rundir)) self._dbinvent.data["grid"] = self.grid self.updateInvent() def getFESOMDsets(conf): """Create dummy tables for displaying""" out = [] out.append(type("vertices_TEMPLATE", (FESOMverticesBase, ), {})) out.append(type("triangles_TEMPLATE", (FESOMtinBase, ), {})) out.append(type("run_TEMPLATE_g_TEMPLATE", (FESOMRunBase, ), {})) return out geoslurpCatalogue.addDatasetFactory(getFESOMDsets)
self._dbinvent.data["Description"] = "ArcticDEM raster table" def pull(self, intersect=None): # download the entire mosaic domain in one tif if self.res in ['1km', '500m', '100m']: rasteruri = http( "http://data.pgc.umn.edu/elev/dem/setsm/ArcticDEM/mosaic/v3.0/" + self.res + "/" + self.rasterfile, lastmod=datetime(2018, 9, 26)) rasterfileuri, upd = rasteruri.download(self.srcdir, check=False) #download only those tiles which are needed def getArcticDems(conf): out = [] for res in ['1km', '500m', '100m']: out.append( type("arcticdem_mosaic_" + res + "_v3", (ArcticDemRasterBase, ), { "res": res, "tiles": [100, 100] })) # out.append(type("arcticdem_mosaic_"+res+"_v3", (ArcticDemRasterBase,), {"res":res})) return out #register datasets geoslurpCatalogue.addDataset(Arcticdemindex) geoslurpCatalogue.addDatasetFactory(getArcticDems)
newfiles=self.retainnewUris([UriFile(file) for file in findFiles(rundir,pattern)]) for uri in newfiles: meta=orasMetaExtractor(uri) if not meta: #don't register empty entries continue self.addEntry(meta) self._dbinvent.data["Description"]="ORAS5 output data table" self.setDataDir(os.path.abspath(rundir)) self._dbinvent.data["grid"]="025" self.updateInvent() #%% ============================================================================= # dataset factory # ============================================================================= def getOrasDsets(conf): """Create dummy tables for displaying""" out=[] out.append(type("vertices_TEMPLATE_TEMPLATE", (orasVerticesBase,), {})) # 'vertices_oras5_025' out.append(type("run_TEMPLATE_TEMPLATE_TEMPLATE_TEMPLATE", (orasRunBase,), {})) # 'run_oras5_temp_opa0_025' return out geoslurpCatalogue.addDatasetFactory(getOrasDsets)
cachedir = conf.getCacheDir("githubcache") try: cred = conf.authCred("github", ['oauthtoken']) token = cred.oauthtoken except: token = None # import pdb;pdb.set_trace() catalog = cachedGithubCatalogue(reponame, cachedir=cachedir, commitsha=commitsha, gfilter=ghfilter({ "type": "blob", "path": "\.geojson" }), gfollowfilter=ghfilter({ "type": "tree", "path": "geojson" }), ghtoken=token) out = [] #create a list of datasets for entry in catalog["datasets"]: clsname = os.path.basename(entry["path"]).split(".")[0] out.append(NaturalEarthClassFactory(clsname, entry)) return out geoslurpCatalogue.addDatasetFactory(getNaturalEarthDsets)
""" Register the drainage divides""" slurplogger().info("Registering %s" % self.name) #possibly empty table self.truncateTable() fname = os.path.join(self.cacheDir(), self.fbase + ".gz") #loop over polygon entries for dicentry in IceSatPolygons(fname): self.addEntry(dicentry) self.updateInvent() def DrainDivClassFactory(clsName, fbase): table = type(clsName + "Table", (draindivTBase, ), {}) return type(clsName, (IceSatDDivBase, ), {"fbase": fbase, "table": table}) def getDrainDivDsets(conf): clslookup = { "antarc_ddiv_icesat": "Ant_Full_DrainageSystem_Polygons.txt", "green_ddiv_icesat": "GrnDrainageSystems_Ekholm.txt", "antarc_ddiv_icesat_grnd": "Ant_Grounded_DrainageSystem_Polygons.txt" } out = [] for clsName, fbase in clslookup.items(): out.append(DrainDivClassFactory(clsName, fbase)) return out geoslurpCatalogue.addDatasetFactory(getDrainDivDsets)