def _check_realm(self):
print("\nChecking for", self.val,"realm in CMIP6 database . . .")
with HidePrint():
realm_av = CMIP6DB().available_realmns()
if self.val not in realm_av:
n=0
m_u=[x.upper() for x in realm_av]
print('\nCannot find realm.\nLooking for other options . . .')
for s in m_u:
if self.val.upper() in s:
n=n+1
s = realm_av[m_u.index(s)]
print('\nOption ', n, s)
if (n==0):
raise ValueError()
else:
raise Exception()
else:
print(color.LBLUE+'Found:'+color.END,self.val,'realm.')
return self.val
def _check_var(self):
print("\nChecking for", self.val,"variable in CMIP6 database . . .")
with HidePrint():
var_av = CMIP6DB().CMIP6_variables()
if self.val not in var_av:
n=0
m_u=[x.upper() for x in var_av]
print('\nCannot find variable.\nLooking for other options . . .')
for s in m_u:
if self.val.upper() in s:
n=n+1
s = var_av[m_u.index(s)]
LN = CMIP6DB()._get_longName(str(s))
print('\nOption ', n, s,'(',LN,')')
if (n==0):
raise ValueError()
else:
raise Exception()
else:
print(color.LBLUE+'Found:'+color.END,self.val,'variable.')
return self.val
def DownloadCmip6(**kwargs):
global startTime
_var = kwargs.get('variable', None)
_mod = kwargs.get('model', None)
_exp = kwargs.get('experiment', None)
_freq = kwargs.get('frequency', None)
_realm = kwargs.get('realm', None)
_check = kwargs.get('check', None)
rlzn = kwargs.get('rlzn', None)
path = kwargs.get('path', None)
skip = kwargs.get('skip', None)
search=SearchDB()
if (_check == 'Yes') or (_check == 'yes'):
search._set_check('Yes')
else:
search._set_check('No')
if (_mod != None):
try:
search.model=_mod
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_mod)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_exp != None):
try:
search.experiment=_exp
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_exp)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_var != None):
try:
search.variable=_var
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_var)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_freq != None):
try:
search.frequency=_freq
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_freq)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_realm != None):
try:
search.realm=_realm
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_realm)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
print("\nFinding server . . .")
links = search.get_links(0)
if (links == []):
print('\n'+color.LRED+'<<Invalid search items!>>'+color.END)
print('\n'+color.UNDERLINE+color.BOLD+'TIPS 1:'+color.END+' Use the check (-c) argument to check your inputs.'+color.END)
print('\n'+color.UNDERLINE+color.BOLD+'TIPS 2:'+color.END+' Use CMIP6DB module to look for currently available '
'models/experiments/variables and so on . . .')
raise SystemExit
if (skip!=None):
links=_get_skipped_links(links,skip)
if (rlzn != None):
all_rlzn = _realizations(links)._all_realizations()
new_links = _get_rlzn_links(rlzn,all_rlzn,links)
unused_links=list(set(links)-set(new_links))
links=new_links
if (path == None):
dir_path = _dir_path()._make_dir()
else:
dir_path = Path(path)
if not os.path.isdir(dir_path):
print('creating ', dir_path)
os.makedirs(str(dir_path))
os.chdir(dir_path)
n = 0
m = 0
manual = 0
passed_urls=[]
for url in links:
startTime = time.time()
try:
dl_cmip6(url, dir_path)
n=n+1
passed_urls.append(url)
except TooSlowException:
print("Removing file . . .\n")
os.remove(url.split('/')[len(url.split('/'))-1])
m=m+1
startTime = time.time()
except KeyboardInterrupt:
print("\nInterrupted! Removing file . . .\n")
os.remove(url.split('/')[len(url.split('/'))-1])
break
except urllib.error.HTTPError:
m=m+1
manual=manual+1
print("\n"+color.RED+"<<401 Unauthorized: restricted access!!>>"+color.END+"\n")
print(color.UNDERLINE+"From ESGF:"+color.END+" Before you can download this data, you have to join a data access control group \nsince acknowledgement of a policy is a condition for this data download.")
print("\nRequires registration/manual download . . . :(")
pass
except:
m=m+1
os.remove(url.split('/')[len(url.split('/'))-1])
pass
print("\nFinished downloading.")
print("\n\nDownloaded ",n," out of ",n+m," files.")
if (m>0):
print("\n\nRe-run the script for the missing files.")
if (manual>0):
print("\n\n",manual," files require an ESGF account/openID.")
print("\nwget script created for these files!\nUse it with your openid/password >> 'wget_script -H'")
with HidePrint():
search.get_links(1)
_manual_wget(passed_urls,unused_links)