def __download_nowcast(self):
names = self.__download(self._html + self._nowcast)
for name in names:
if name.find("2D") > 0:
move_iso(name, "Models/GLCFS/Nowcast-2D")
else:
move_iso(name, "Models/GLCFS/Nowcast-3D")
def download_isos(self):
"""
Should download all Agg files from the html and put them in the correct glos catalog folder
"""
resp = download(self._html)
tree = etree.HTML(resp)
datasets = list()
files = dict()
# get all of the datasets to download
for elm in tree.findall(".//a[@href]"):
for key in elm.attrib.keys():
val = elm.attrib[key]
if val.find("dataset") > 0:
ds = val[val.find("=") + 1 :]
datasets.append(ds)
# download isos
for ds in datasets:
url = self._iso + ds
files[ds] = download_iso(url, catalog=self._html, dataset=ds)
# move them
for key in files.keys():
move_iso(files[key], self._iso_dir)
def __download_forecast(self):
names = self.__download(self._html + self._forecast)
for name in names:
nname = list(name.partition("_-_"))
nname.append(nname[2])
nname[2] = "Forecast_-_"
nname = "".join(nname)
if nname.find("2D") > 0:
move_iso(name, "Models/GLCFS/Forecast-2D", nname)
else:
move_iso(name, "Models/GLCFS/Forecast-3D", nname)
def download_isos(self):
# get datasets
resp = download(self._html)
tree = etree.HTML(resp)
for a in tree.findall('.//a[@href]'):
dataset = a.attrib.get('href')
# will have to correct for a typo in the address
dataset = dataset.replace('Lastest','Latest')
if dataset.find('dataset') > 0:
dataset = dataset[dataset.find('=')+1:]
# get iso
if dataset.find('Agg') > 0:
url = self._agg + dataset + '.nc'
else:
url = self._latest + dataset + '.nc'
fname = download_iso(url, catalog=self._html, dataset=dataset)
# move iso
move_iso(fname, self._iso)
def download_isos(self):
resp = download(self._html+self._sources)
tree = etree.HTML(resp)
for source in tree.findall(".//li"):
if source.text in self._iso_dirs:
# get list of files in source
dirlist = download(self._html+source.text+'/list.html')
dirtree = etree.HTML(dirlist)
for iso in dirtree.findall('.//li'):
sname = download_iso(self._html+source.text+'/'+iso.text.strip())
if source.text != 'STORET':
move_iso(sname, self._iso_dirs[source.text])
elif sname is not None:
# storet needs to be compressed before being moved as a single archive
if path.exists(self._storet_zip_dir + sname):
remove(self._storet_zip_dir + sname)
move('./pyiso/iso_tmp/' + sname, self._storet_zip_dir)
if source.text == 'STORET':
# archive the storet directory in iso_tmp and move it to the ISOs directory
f = file(path.abspath('./pyiso/iso_tmp/storet.zip'), 'w')
with ZipFile(f, 'w') as zip_file:
fileiter = (f
for root, _, files in walk(self._storet_zip_dir)
for f in files)
xmlfilter = (f for f in fileiter if f.endswith('.xml'))
for xml in xmlfilter:
zip_file.write(self._storet_zip_dir + xml, arcname=xml)
# move the zip file
dest = '../ISOs/' + self._iso_dirs[source.text] + '/storet.zip'
if path.exists(dest):
remove(dest)
move('./pyiso/iso_tmp/storet.zip', dest)
def __download_forcing(self):
# get all of the catalogs at the forcing url
fnames = self.__download(self._html + self._forcing)
for name in fnames:
move_iso(name, "Models/GLCFS/Forcing/")
