def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings # Ranger 3 selects = [".*RANGER3"] isos = ThreddsCollector("http://tds.glos.us/thredds/ranger3.html", selects=selects).run() download_path = os.path.join(base_download_path, "RANGER3") XmlDownloader.run(isos, download_path) # GLERL selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/NBS.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLERL", "NBS") XmlDownloader.run(isos, download_path) selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/ATMO.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLERL", "ATMO") XmlDownloader.run(isos, download_path) selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/PRECIP.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLERL", "PRECIP") XmlDownloader.run(isos, download_path) selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/AIRTEMPS.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS") XmlDownloader.run(isos, download_path)
def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings isos = GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run() download_path = os.path.join(base_download_path, "GeoNetwork") XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.namer, modifier=GeoNetworkCollector.modifier)
def test_thredds_default_skips(self): # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] isos = ThreddsCollector("http://tds.glos.us:8080/thredds/hecwfs/hecwfs.html", selects=selects).run() # 2 ISOs (Nowcast and latest Forecast) #assert len(isos) == 2 XmlDownloader.run(isos, self.scratch) #assert os.path.isfile(os.path.join(self.scratch, "HECWFS-Latest-Forecast.nc.xml")) #assert os.path.isfile(os.path.join(self.scratch, "HECWFS-Nowcast-Agg.nc.xml"))
def test_thredds_custom_skips(self): # We only want the Agg datasets selects = [".*SST-Agg.*"] # Don't process the "files/" lists skips = ["files/"] isos = ThreddsCollector("http://tds.glos.us:8080/thredds/mtri/aoc.html", selects=selects, skips=skips).run() # 5 Lake ISOs #print isos #assert len(isos) == 5 XmlDownloader.run(isos, self.scratch)
def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings # CIA selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glc/wateruse.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLC") XmlDownloader.run(isos, download_path)
def geonetwork_collector(base_download_path): # read isos not equal to 260, suspect crashing is occurring due to this. isos = [ i for i in GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run() if i[-3:] != '260' ] download_path = os.path.join(base_download_path, "GeoNetwork") XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.uuid_namer, modifier=GeoNetworkCollector.modifier)
def run_downloader(selects, url, dlpath, **kwargs): """ Selects is list of selectors Url is url of base catalog page dlpath is download path kwargs is any other kwargs to ThreddsCollector """ print "Downloading", url, "to", dlpath try: isos = ThreddsCollector(url, selects=selects, **kwargs).run() XmlDownloader.run(isos, dlpath) except Exception as e: print >>sys.stderr, "Problem with", url, "ex:", e
def run_downloader(selects, url, dlpath, **kwargs): """ Selects is list of selectors Url is url of base catalog page dlpath is download path kwargs is any other kwargs to ThreddsCollector """ print "Downloading", url, "to", dlpath try: isos = ThreddsCollector(url, selects=selects, **kwargs).run() XmlDownloader.run(isos, dlpath) except Exception as e: print >> sys.stderr, "Problem with", url, "ex:", e
def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings # We only want the Agg selects selects = [".*SST-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/aoc.html", download_path) # We only want the Agg selects selects = [".*CHL-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/chl.html", download_path) # We only want the Agg selects selects = [".*NC-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/natcolor.html", download_path) # MTRI CDOM, DOC and SM Aggregates selects = [".*CDOM-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/cdom.html", download_path) selects = [".*DOC-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/doc.html", download_path) selects = [".*SM-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/sm.html", download_path) # We only want the all year best selects selects = [ ".*2D_best.*", ".*3D_best.*", ".*2D_-_All_Years_best.*", ".*3D_-_All_Years_best.*" ] download_path = os.path.join(base_download_path, "Models", "GLCFS") run_downloader(selects, "http://tds.glos.us/thredds/glcfs/glcfs.html", download_path) # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] # Don't process the "files/" lists skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"] download_path = os.path.join(base_download_path, "Models", "HECWFS") run_downloader(selects, "http://tds.glos.us/thredds/hecwfs/hecwfs.html", download_path, skips=skips) # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] # Don't process the "files/" lists skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"] download_path = os.path.join(base_download_path, "Models", "SLRFVM") run_downloader(selects, "http://tds.glos.us/thredds/slrfvm/slrfvm.html", download_path, skips=skips) # RANGER3 selects = [".*RANGER3"] download_path = os.path.join(base_download_path, "RANGER3") run_downloader(selects, "http://tds.glos.us/thredds/ranger3.html", download_path) # GLERL selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "NBS") run_downloader(selects, "http://tds.glos.us/thredds/glerl/NBS.html", download_path) selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "ATMO") run_downloader(selects, "http://tds.glos.us/thredds/glerl/ATMO.html", download_path) selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "PRECIP") run_downloader(selects, "http://tds.glos.us/thredds/glerl/PRECIP.html", download_path) selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS") run_downloader(selects, "http://tds.glos.us/thredds/glerl/AIRTEMPS.html", download_path) # water levels selects = [".*"] download_path = os.path.join(base_download_path, "WaterLevels") run_downloader(selects, "http://tds.glos.us/thredds/water_levels.html", download_path) #UMD Glider selects = [".*"] download_path = os.path.join(base_download_path, "UMDGlider") run_downloader(selects, "http://tds.glos.us/thredds/glos_glider.html", download_path) # GLOP selects = [".*"] download_path = os.path.join(base_download_path, "GLOP") run_downloader(selects, "http://tds.glos.us/thredds/glop/glop.html", download_path) # CIA selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glc/wateruse.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLC") XmlDownloader.run(isos, download_path) # MTRI Water Temp selects = [".*"] download_path = os.path.join(base_download_path, "Thermistor") run_downloader(selects, "http://tds.glos.us/thredds/mtri/water_temp_in_situ.html", download_path)
def test_geonetwork(self): scratch_dir = os.path.join(os.path.dirname(__file__),'scratch') isos = GeoNetworkCollector("http://data.glos.us/metadata").run() XmlDownloader.run(isos, scratch_dir, namer=GeoNetworkCollector.namer, modifier=GeoNetworkCollector.modifier)
def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings # We only want the Agg selects selects = [".*SST-Agg", "SST-.*-Agg"] isos = ThreddsCollector("http://tds.maracoos.org/thredds/SST.html", selects=selects, debug=True).run() download_path = os.path.join(base_download_path, "Satellite") XmlDownloader.run(isos, download_path) # We only want the Agg selects selects = [".*MODIS-Agg", ".*MODIS-.*-Agg"] isos = ThreddsCollector("http://tds.maracoos.org/thredds/MODIS.html", selects=selects, debug=True).run() download_path = os.path.join(base_download_path, "Satellite") XmlDownloader.run(isos, download_path) # We only want the Agg selects selects = [".*Forecast_Aggregation_Collection_best.ncd"] isos = ThreddsCollector("http://tds.maracoos.org/thredds/Weatherflow.html", selects=selects, debug=True).run() download_path = os.path.join(base_download_path, "Models", "Weatherflow") XmlDownloader.run(isos, download_path) # We only want the all year best selects selects = ["Rutgers_WRF_.*"] isos = ThreddsCollector("http://tds.maracoos.org/thredds/RUCOOL.html", selects=selects, debug=True).run() download_path = os.path.join(base_download_path, "Models", "RUWRF") XmlDownloader.run(isos, download_path) # We want everything that isn't skipped selects = [".*"] skips = ThreddsCollector.SKIPS isos = ThreddsCollector( "http://colossus.dl.stevens-tech.edu/thredds/catalog.html", selects=selects, skips=skips, debug=True ).run() download_path = os.path.join(base_download_path, "Models", "NYHOPS") XmlDownloader.run(isos, download_path) # We want everything that isn't skipped selects = [".*"] skips = [".*Forcing Files.*"] + ThreddsCollector.SKIPS isos = ThreddsCollector( "http://tds.marine.rutgers.edu/thredds/roms/espresso/catalog.html", selects=selects, skips=skips, debug=True ).run() download_path = os.path.join(base_download_path, "Models", "ESPRESSO") XmlDownloader.run(isos, download_path) # We want everything that isn't skipped selects = [".*"] skips = ThreddsCollector.SKIPS isos = ThreddsCollector( "http://aqua.smast.umassd.edu:8080/thredds/catalog/pe_tnbc/fmrc/catalog.html", selects=selects, skips=skips, debug=True, ).run() download_path = os.path.join(base_download_path, "Models", "HOPS_TNBC") XmlDownloader.run(isos, download_path) # We want everything that isn't skipped selects = [".*"] skips = ThreddsCollector.SKIPS isos = ThreddsCollector( "http://aqua.smast.umassd.edu:8080/thredds/catalog/pe_shelf_ass/fmrc/catalog.html", selects=selects, skips=skips, debug=True, ).run() download_path = os.path.join(base_download_path, "Models", "HOPS_PESHELF_ASS") XmlDownloader.run(isos, download_path) # We want the Best datasets selects = [".*best.ncd"] isos = ThreddsCollector("http://geoport.whoi.edu/thredds/COAWST_catalog.html", selects=selects, debug=True).run() download_path = os.path.join(base_download_path, "Models", "COAWST") XmlDownloader.run(isos, download_path) # We want everything that isn't skipped selects = ["macoora6km_codar_archive", ".*best.ncd"] isos = ThreddsCollector( "http://tds.marine.rutgers.edu/thredds/cool/codar/cat_totals.html", selects=selects, debug=True ).run() download_path = os.path.join(base_download_path, "Hfradar") XmlDownloader.run(isos, download_path)
def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings # We only want the Agg selects selects = [".*SST-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/aoc.html", download_path) # We only want the Agg selects selects = [".*CHL-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/chl.html", download_path) # We only want the Agg selects selects = [".*NC-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/natcolor.html", download_path) # MTRI CDOM, DOC and SM Aggregates selects = [".*CDOM-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/cdom.html", download_path) selects = [".*DOC-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/doc.html", download_path) selects = [".*SM-Agg"] download_path = os.path.join(base_download_path, "Satellite") run_downloader(selects, "http://tds.glos.us/thredds/mtri/sm.html", download_path) # We only want the all year best selects selects = [".*2D_best.*", ".*3D_best.*", ".*2D_-_All_Years_best.*", ".*3D_-_All_Years_best.*"] download_path = os.path.join(base_download_path, "Models", "GLCFS") run_downloader(selects, "http://tds.glos.us/thredds/glcfs/glcfs.html", download_path) # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] # Don't process the "files/" lists skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"] download_path = os.path.join(base_download_path, "Models", "HECWFS") run_downloader(selects, "http://tds.glos.us/thredds/hecwfs/hecwfs.html", download_path, skips=skips) # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] # Don't process the "files/" lists skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"] download_path = os.path.join(base_download_path, "Models", "SLRFVM") run_downloader(selects, "http://tds.glos.us/thredds/slrfvm/slrfvm.html", download_path, skips=skips) # RANGER3 selects = [".*RANGER3"] download_path = os.path.join(base_download_path, "RANGER3") run_downloader(selects, "http://tds.glos.us/thredds/ranger3.html", download_path) # GLERL selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "NBS") run_downloader(selects, "http://tds.glos.us/thredds/glerl/NBS.html", download_path) selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "ATMO") run_downloader(selects, "http://tds.glos.us/thredds/glerl/ATMO.html", download_path) selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "PRECIP") run_downloader(selects, "http://tds.glos.us/thredds/glerl/PRECIP.html", download_path) selects = [".*"] download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS") run_downloader(selects, "http://tds.glos.us/thredds/glerl/AIRTEMPS.html", download_path) # water levels selects = [".*"] download_path = os.path.join(base_download_path, "WaterLevels") run_downloader(selects, "http://tds.glos.us/thredds/water_levels.html", download_path) #UMD Glider selects = [".*"] download_path = os.path.join(base_download_path, "UMDGlider") run_downloader(selects, "http://tds.glos.us/thredds/glos_glider.html", download_path) # GLOP selects = [".*"] download_path = os.path.join(base_download_path, "GLOP") run_downloader(selects, "http://tds.glos.us/thredds/glop/glop.html", download_path) # CIA selects = [".*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glc/wateruse.html", selects=selects).run() download_path = os.path.join(base_download_path, "GLC") XmlDownloader.run(isos, download_path) # MTRI Water Temp selects = [".*"] download_path = os.path.join(base_download_path, "Thermistor") run_downloader(selects, "http://tds.glos.us/thredds/mtri/water_temp_in_situ.html", download_path)
def geonetwork_collector(base_download_path): # read isos not equal to 260, suspect crashing is occurring due to this. isos = [i for i in GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run() if i[-3:] != '260'] download_path = os.path.join(base_download_path, "GeoNetwork") XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.uuid_namer, modifier=GeoNetworkCollector.modifier)
def main(base_download_path): # selects: The ID in THREDDS needs to contain one of these strings to be identified. # skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings # We only want the Agg selects selects = [".*SST-Agg"] isos = ThreddsCollector("http://tds.glos.us/thredds/mtri/aoc.html", selects=selects).run() download_path = os.path.join(base_download_path, "Satellite") XmlDownloader.run(isos, download_path) # We only want the Agg selects selects = [".*CHL-Agg"] isos = ThreddsCollector("http://tds.glos.us/thredds/mtri/chl.html", selects=selects).run() download_path = os.path.join(base_download_path, "Satellite") XmlDownloader.run(isos, download_path) # We only want the Agg selects selects = [".*NC-Agg"] isos = ThreddsCollector("http://tds.glos.us/thredds/mtri/natcolor.html", selects=selects).run() download_path = os.path.join(base_download_path, "Satellite") XmlDownloader.run(isos, download_path) # We only want the all year best selects selects = [".*2D_best.*", ".*3D_best.*", ".*2D_-_All_Years_best.*", ".*3D_-_All_Years_best.*"] isos = ThreddsCollector("http://tds.glos.us/thredds/glcfs/glcfs.html", selects=selects).run() download_path = os.path.join(base_download_path, "Models", "GLCFS") XmlDownloader.run(isos, download_path) # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] # Don't process the "files/" lists skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"] isos = ThreddsCollector("http://tds.glos.us/thredds/hecwfs/hecwfs.html", selects=selects, skips=skips).run() download_path = os.path.join(base_download_path, "Models", "HECWFS") XmlDownloader.run(isos, download_path) # We only want the Agg and Latest selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"] # Don't process the "files/" lists skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"] isos = ThreddsCollector("http://tds.glos.us/thredds/slrfvm/slrfvm.html", selects=selects, skips=skips).run() download_path = os.path.join(base_download_path, "Models", "SLRFVM") XmlDownloader.run(isos, download_path) isos = GeoNetworkCollector("http://data.glos.us/metadata").run() download_path = os.path.join(base_download_path, "GeoNetwork") XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.namer, modifier=GeoNetworkCollector.modifier)