def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# Ranger 3
selects = [".*RANGER3"]
isos = ThreddsCollector("http://tds.glos.us/thredds/ranger3.html", selects=selects).run()
download_path = os.path.join(base_download_path, "RANGER3")
XmlDownloader.run(isos, download_path)
# GLERL
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/NBS.html", selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "NBS")
XmlDownloader.run(isos, download_path)
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/ATMO.html", selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "ATMO")
XmlDownloader.run(isos, download_path)
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/PRECIP.html", selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "PRECIP")
XmlDownloader.run(isos, download_path)
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/AIRTEMPS.html", selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS")
XmlDownloader.run(isos, download_path)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
isos = GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run()
download_path = os.path.join(base_download_path, "GeoNetwork")
XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.namer, modifier=GeoNetworkCollector.modifier)
def test_thredds_default_skips(self):
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
isos = ThreddsCollector("http://tds.glos.us:8080/thredds/hecwfs/hecwfs.html", selects=selects).run()
# 2 ISOs (Nowcast and latest Forecast)
#assert len(isos) == 2
XmlDownloader.run(isos, self.scratch)
#assert os.path.isfile(os.path.join(self.scratch, "HECWFS-Latest-Forecast.nc.xml"))
#assert os.path.isfile(os.path.join(self.scratch, "HECWFS-Nowcast-Agg.nc.xml"))
def test_thredds_custom_skips(self):
# We only want the Agg datasets
selects = [".*SST-Agg.*"]
# Don't process the "files/" lists
skips = ["files/"]
isos = ThreddsCollector("http://tds.glos.us:8080/thredds/mtri/aoc.html", selects=selects, skips=skips).run()
# 5 Lake ISOs
#print isos
#assert len(isos) == 5
XmlDownloader.run(isos, self.scratch)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# CIA
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glc/wateruse.html", selects=selects).run()
download_path = os.path.join(base_download_path, "GLC")
XmlDownloader.run(isos, download_path)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
isos = GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run()
download_path = os.path.join(base_download_path, "GeoNetwork")
XmlDownloader.run(isos,
download_path,
namer=GeoNetworkCollector.namer,
modifier=GeoNetworkCollector.modifier)
def geonetwork_collector(base_download_path):
# read isos not equal to 260, suspect crashing is occurring due to this.
isos = [
i
for i in GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run()
if i[-3:] != '260'
]
download_path = os.path.join(base_download_path, "GeoNetwork")
XmlDownloader.run(isos,
download_path,
namer=GeoNetworkCollector.uuid_namer,
modifier=GeoNetworkCollector.modifier)
def run_downloader(selects, url, dlpath, **kwargs):
"""
Selects is list of selectors
Url is url of base catalog page
dlpath is download path
kwargs is any other kwargs to ThreddsCollector
"""
print "Downloading", url, "to", dlpath
try:
isos = ThreddsCollector(url, selects=selects, **kwargs).run()
XmlDownloader.run(isos, dlpath)
except Exception as e:
print >>sys.stderr, "Problem with", url, "ex:", e
def run_downloader(selects, url, dlpath, **kwargs):
"""
Selects is list of selectors
Url is url of base catalog page
dlpath is download path
kwargs is any other kwargs to ThreddsCollector
"""
print "Downloading", url, "to", dlpath
try:
isos = ThreddsCollector(url, selects=selects, **kwargs).run()
XmlDownloader.run(isos, dlpath)
except Exception as e:
print >> sys.stderr, "Problem with", url, "ex:", e
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# Ranger 3
selects = [".*RANGER3"]
isos = ThreddsCollector("http://tds.glos.us/thredds/ranger3.html",
selects=selects).run()
download_path = os.path.join(base_download_path, "RANGER3")
XmlDownloader.run(isos, download_path)
# GLERL
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/NBS.html",
selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "NBS")
XmlDownloader.run(isos, download_path)
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/ATMO.html",
selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "ATMO")
XmlDownloader.run(isos, download_path)
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/PRECIP.html",
selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "PRECIP")
XmlDownloader.run(isos, download_path)
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glerl/AIRTEMPS.html",
selects=selects).run()
download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS")
XmlDownloader.run(isos, download_path)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# We only want the Agg selects
selects = [".*SST-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/aoc.html",
download_path)
# We only want the Agg selects
selects = [".*CHL-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/chl.html",
download_path)
# We only want the Agg selects
selects = [".*NC-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/natcolor.html",
download_path)
# MTRI CDOM, DOC and SM Aggregates
selects = [".*CDOM-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/cdom.html",
download_path)
selects = [".*DOC-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/doc.html",
download_path)
selects = [".*SM-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/sm.html",
download_path)
# We only want the all year best selects
selects = [
".*2D_best.*", ".*3D_best.*", ".*2D_-_All_Years_best.*",
".*3D_-_All_Years_best.*"
]
download_path = os.path.join(base_download_path, "Models", "GLCFS")
run_downloader(selects, "http://tds.glos.us/thredds/glcfs/glcfs.html",
download_path)
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
# Don't process the "files/" lists
skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"]
download_path = os.path.join(base_download_path, "Models", "HECWFS")
run_downloader(selects,
"http://tds.glos.us/thredds/hecwfs/hecwfs.html",
download_path,
skips=skips)
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
# Don't process the "files/" lists
skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"]
download_path = os.path.join(base_download_path, "Models", "SLRFVM")
run_downloader(selects,
"http://tds.glos.us/thredds/slrfvm/slrfvm.html",
download_path,
skips=skips)
# RANGER3
selects = [".*RANGER3"]
download_path = os.path.join(base_download_path, "RANGER3")
run_downloader(selects, "http://tds.glos.us/thredds/ranger3.html",
download_path)
# GLERL
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "NBS")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/NBS.html",
download_path)
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "ATMO")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/ATMO.html",
download_path)
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "PRECIP")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/PRECIP.html",
download_path)
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/AIRTEMPS.html",
download_path)
# water levels
selects = [".*"]
download_path = os.path.join(base_download_path, "WaterLevels")
run_downloader(selects, "http://tds.glos.us/thredds/water_levels.html",
download_path)
#UMD Glider
selects = [".*"]
download_path = os.path.join(base_download_path, "UMDGlider")
run_downloader(selects, "http://tds.glos.us/thredds/glos_glider.html",
download_path)
# GLOP
selects = [".*"]
download_path = os.path.join(base_download_path, "GLOP")
run_downloader(selects, "http://tds.glos.us/thredds/glop/glop.html",
download_path)
# CIA
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glc/wateruse.html",
selects=selects).run()
download_path = os.path.join(base_download_path, "GLC")
XmlDownloader.run(isos, download_path)
# MTRI Water Temp
selects = [".*"]
download_path = os.path.join(base_download_path, "Thermistor")
run_downloader(selects,
"http://tds.glos.us/thredds/mtri/water_temp_in_situ.html",
download_path)
def test_geonetwork(self):
scratch_dir = os.path.join(os.path.dirname(__file__),'scratch')
isos = GeoNetworkCollector("http://data.glos.us/metadata").run()
XmlDownloader.run(isos, scratch_dir, namer=GeoNetworkCollector.namer, modifier=GeoNetworkCollector.modifier)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# We only want the Agg selects
selects = [".*SST-Agg", "SST-.*-Agg"]
isos = ThreddsCollector("http://tds.maracoos.org/thredds/SST.html", selects=selects, debug=True).run()
download_path = os.path.join(base_download_path, "Satellite")
XmlDownloader.run(isos, download_path)
# We only want the Agg selects
selects = [".*MODIS-Agg", ".*MODIS-.*-Agg"]
isos = ThreddsCollector("http://tds.maracoos.org/thredds/MODIS.html", selects=selects, debug=True).run()
download_path = os.path.join(base_download_path, "Satellite")
XmlDownloader.run(isos, download_path)
# We only want the Agg selects
selects = [".*Forecast_Aggregation_Collection_best.ncd"]
isos = ThreddsCollector("http://tds.maracoos.org/thredds/Weatherflow.html", selects=selects, debug=True).run()
download_path = os.path.join(base_download_path, "Models", "Weatherflow")
XmlDownloader.run(isos, download_path)
# We only want the all year best selects
selects = ["Rutgers_WRF_.*"]
isos = ThreddsCollector("http://tds.maracoos.org/thredds/RUCOOL.html", selects=selects, debug=True).run()
download_path = os.path.join(base_download_path, "Models", "RUWRF")
XmlDownloader.run(isos, download_path)
# We want everything that isn't skipped
selects = [".*"]
skips = ThreddsCollector.SKIPS
isos = ThreddsCollector(
"http://colossus.dl.stevens-tech.edu/thredds/catalog.html", selects=selects, skips=skips, debug=True
).run()
download_path = os.path.join(base_download_path, "Models", "NYHOPS")
XmlDownloader.run(isos, download_path)
# We want everything that isn't skipped
selects = [".*"]
skips = [".*Forcing Files.*"] + ThreddsCollector.SKIPS
isos = ThreddsCollector(
"http://tds.marine.rutgers.edu/thredds/roms/espresso/catalog.html", selects=selects, skips=skips, debug=True
).run()
download_path = os.path.join(base_download_path, "Models", "ESPRESSO")
XmlDownloader.run(isos, download_path)
# We want everything that isn't skipped
selects = [".*"]
skips = ThreddsCollector.SKIPS
isos = ThreddsCollector(
"http://aqua.smast.umassd.edu:8080/thredds/catalog/pe_tnbc/fmrc/catalog.html",
selects=selects,
skips=skips,
debug=True,
).run()
download_path = os.path.join(base_download_path, "Models", "HOPS_TNBC")
XmlDownloader.run(isos, download_path)
# We want everything that isn't skipped
selects = [".*"]
skips = ThreddsCollector.SKIPS
isos = ThreddsCollector(
"http://aqua.smast.umassd.edu:8080/thredds/catalog/pe_shelf_ass/fmrc/catalog.html",
selects=selects,
skips=skips,
debug=True,
).run()
download_path = os.path.join(base_download_path, "Models", "HOPS_PESHELF_ASS")
XmlDownloader.run(isos, download_path)
# We want the Best datasets
selects = [".*best.ncd"]
isos = ThreddsCollector("http://geoport.whoi.edu/thredds/COAWST_catalog.html", selects=selects, debug=True).run()
download_path = os.path.join(base_download_path, "Models", "COAWST")
XmlDownloader.run(isos, download_path)
# We want everything that isn't skipped
selects = ["macoora6km_codar_archive", ".*best.ncd"]
isos = ThreddsCollector(
"http://tds.marine.rutgers.edu/thredds/cool/codar/cat_totals.html", selects=selects, debug=True
).run()
download_path = os.path.join(base_download_path, "Hfradar")
XmlDownloader.run(isos, download_path)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# We only want the Agg selects
selects = [".*SST-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/aoc.html", download_path)
# We only want the Agg selects
selects = [".*CHL-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/chl.html", download_path)
# We only want the Agg selects
selects = [".*NC-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/natcolor.html", download_path)
# MTRI CDOM, DOC and SM Aggregates
selects = [".*CDOM-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/cdom.html", download_path)
selects = [".*DOC-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/doc.html", download_path)
selects = [".*SM-Agg"]
download_path = os.path.join(base_download_path, "Satellite")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/sm.html", download_path)
# We only want the all year best selects
selects = [".*2D_best.*", ".*3D_best.*", ".*2D_-_All_Years_best.*", ".*3D_-_All_Years_best.*"]
download_path = os.path.join(base_download_path, "Models", "GLCFS")
run_downloader(selects, "http://tds.glos.us/thredds/glcfs/glcfs.html", download_path)
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
# Don't process the "files/" lists
skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"]
download_path = os.path.join(base_download_path, "Models", "HECWFS")
run_downloader(selects, "http://tds.glos.us/thredds/hecwfs/hecwfs.html", download_path, skips=skips)
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
# Don't process the "files/" lists
skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"]
download_path = os.path.join(base_download_path, "Models", "SLRFVM")
run_downloader(selects, "http://tds.glos.us/thredds/slrfvm/slrfvm.html", download_path, skips=skips)
# RANGER3
selects = [".*RANGER3"]
download_path = os.path.join(base_download_path, "RANGER3")
run_downloader(selects, "http://tds.glos.us/thredds/ranger3.html", download_path)
# GLERL
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "NBS")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/NBS.html", download_path)
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "ATMO")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/ATMO.html", download_path)
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "PRECIP")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/PRECIP.html", download_path)
selects = [".*"]
download_path = os.path.join(base_download_path, "GLERL", "AIRTEMPS")
run_downloader(selects, "http://tds.glos.us/thredds/glerl/AIRTEMPS.html", download_path)
# water levels
selects = [".*"]
download_path = os.path.join(base_download_path, "WaterLevels")
run_downloader(selects, "http://tds.glos.us/thredds/water_levels.html", download_path)
#UMD Glider
selects = [".*"]
download_path = os.path.join(base_download_path, "UMDGlider")
run_downloader(selects, "http://tds.glos.us/thredds/glos_glider.html", download_path)
# GLOP
selects = [".*"]
download_path = os.path.join(base_download_path, "GLOP")
run_downloader(selects, "http://tds.glos.us/thredds/glop/glop.html", download_path)
# CIA
selects = [".*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glc/wateruse.html", selects=selects).run()
download_path = os.path.join(base_download_path, "GLC")
XmlDownloader.run(isos, download_path)
# MTRI Water Temp
selects = [".*"]
download_path = os.path.join(base_download_path, "Thermistor")
run_downloader(selects, "http://tds.glos.us/thredds/mtri/water_temp_in_situ.html", download_path)
def geonetwork_collector(base_download_path):
# read isos not equal to 260, suspect crashing is occurring due to this.
isos = [i for i in GeoNetworkCollector("http://slrfvm.glos.us/geonetwork").run() if i[-3:] != '260']
download_path = os.path.join(base_download_path, "GeoNetwork")
XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.uuid_namer, modifier=GeoNetworkCollector.modifier)
def main(base_download_path):
# selects: The ID in THREDDS needs to contain one of these strings to be identified.
# skips: The LINK path in the actual thredds catalog webpage can't be equal to any of these strings
# We only want the Agg selects
selects = [".*SST-Agg"]
isos = ThreddsCollector("http://tds.glos.us/thredds/mtri/aoc.html", selects=selects).run()
download_path = os.path.join(base_download_path, "Satellite")
XmlDownloader.run(isos, download_path)
# We only want the Agg selects
selects = [".*CHL-Agg"]
isos = ThreddsCollector("http://tds.glos.us/thredds/mtri/chl.html", selects=selects).run()
download_path = os.path.join(base_download_path, "Satellite")
XmlDownloader.run(isos, download_path)
# We only want the Agg selects
selects = [".*NC-Agg"]
isos = ThreddsCollector("http://tds.glos.us/thredds/mtri/natcolor.html", selects=selects).run()
download_path = os.path.join(base_download_path, "Satellite")
XmlDownloader.run(isos, download_path)
# We only want the all year best selects
selects = [".*2D_best.*", ".*3D_best.*", ".*2D_-_All_Years_best.*", ".*3D_-_All_Years_best.*"]
isos = ThreddsCollector("http://tds.glos.us/thredds/glcfs/glcfs.html", selects=selects).run()
download_path = os.path.join(base_download_path, "Models", "GLCFS")
XmlDownloader.run(isos, download_path)
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
# Don't process the "files/" lists
skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"]
isos = ThreddsCollector("http://tds.glos.us/thredds/hecwfs/hecwfs.html", selects=selects, skips=skips).run()
download_path = os.path.join(base_download_path, "Models", "HECWFS")
XmlDownloader.run(isos, download_path)
# We only want the Agg and Latest
selects = [".*Nowcast-Agg.*", ".*Lastest-Forecast.*"]
# Don't process the "files/" lists
skips = [".*Nowcast - Individual Files/", ".*Forecast - Individual Files/"]
isos = ThreddsCollector("http://tds.glos.us/thredds/slrfvm/slrfvm.html", selects=selects, skips=skips).run()
download_path = os.path.join(base_download_path, "Models", "SLRFVM")
XmlDownloader.run(isos, download_path)
isos = GeoNetworkCollector("http://data.glos.us/metadata").run()
download_path = os.path.join(base_download_path, "GeoNetwork")
XmlDownloader.run(isos, download_path, namer=GeoNetworkCollector.namer, modifier=GeoNetworkCollector.modifier)
