def SearchCmip6(**kwargs):
_var = kwargs.get('variable', None)
_mod = kwargs.get('model', None)
_exp = kwargs.get('experiment', None)
_freq = kwargs.get('frequency', None)
_realm = kwargs.get('realm', None)
_check = kwargs.get('check', None)
_desc = kwargs.get('desc', None)
_time = kwargs.get('time', None)
year = kwargs.get('year', None)
module = kwargs.get('module', None)
rlzn = kwargs.get('rlzn', None)
skip = kwargs.get('skip', None)
cr = kwargs.get('cr', None)
if (module == 'on'):
_mod_help()
if (_var == 'show') or (_var == 'Show'):
ModDB = CMIP6DB()
print(ModDB.var_stdName())
if (_mod == 'show') or (_mod == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_models())
if (_exp == 'show') or (_exp == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_experiments())
if (_freq == 'show') or (_freq == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_frequencies())
if (_realm == 'show') or (_realm == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_realmns())
raise SystemExit
search=SearchDB()
if (_check == 'Yes') or (_check == 'yes'):
print('\n'+color.UNDERLINE+color.BOLD+'TIPS:'+color.END+" If you are not sure about what you are looking for use CMIP6DB module \n to look for currently available models/experiments/variables and so on . . ."+color.END)
search._set_check('Yes')
else:
print('\n'+color.UNDERLINE+color.BOLD+'TIPS:'+color.END+" Use the check (-c) argument to check your inputs."+color.END)
search._set_check('No')
if (_mod != None):
try:
search.model=_mod
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_mod)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_exp != None):
try:
search.experiment=_exp
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_exp)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_var != None):
try:
search.variable=_var
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_var)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_freq != None):
try:
search.frequency=_freq
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_freq)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_realm != None):
try:
search.realm=_realm
except ValueError as ve:
print(color.LRED+"\n<<No options available.>>\n\nPlease make sure "+str(_realm)+" exists."+color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
info = search.get_info()
if (year!=None):
all_links = info.links
links=[]
if int(year)>0:
end_year = int(info.year[0])+int(year)
else:
end_year = int(info.year[len(info.year)-1])+int(year)
interested_years=[]
for y in info.year:
if (int(year)>0) and (int(y)-1<end_year):
interested_years.append(y)
elif (int(year)<0) and (end_year<int(y)+1):
interested_years.append(y)
else:
continue
for item in interested_years:
for link in all_links:
if '_'+item in link:
links.append(link)
search.links = links
info = search.get_info()
if (rlzn != None):
links = info.links
all_rlzn = _realizations(links)._all_realizations()
new_links = _get_rlzn_links(rlzn,all_rlzn,links)
search.links=new_links
info = search.get_info()
if (skip!=None):
links = info.links
new_links=_get_skipped_links(links,skip)
search.links=new_links
info = search.get_info()
if (cr!=None):
links = info.links
exps = info.exp
ers=[]
for i in range(len(exps)):
ers1 = [s for s in links if exps[i] in s]
r1 = _realizations(ers1)._all_realizations()
if i==1:
ers2 = [s for s in links if exps[i-1] in s]
r2 = _realizations(ers2)._all_realizations()
elif i>1:
r2 = ers
else:
ers2 = ers1
r2 = _realizations(ers2)._all_realizations()
ers = list(set(r1) & set(r2))
erss=['_r'+str(s)+'i' for s in ers]
links=[s for s in links if any(xs in s for xs in erss)]
info.rlzn = ers
print(color.LGREEN+"\n\n Currently available models based on your search: \n\n"+color.END,info.mod)
print(color.LGREEN+"\nCurrently available variables based on your search: \n\n"+color.END,info.var)
print(color.LGREEN+"\nCurrently available experiments based on your search: \n\n"+color.END,info.exp,"\n\n")
if (cr!=None):
print(color.LGREEN+"\nNumber of files:"+color.END, len(links),"\n\n")
print(color.LGREEN+"\nAvailable common realizations:"+color.END, info.rlzn,"\n\n")
else:
print(color.LGREEN+"\nNumber of files:"+color.END, info.n_files,"\n\n")
print(color.LGREEN+"\nAvailable realizations:"+color.END, info.rlzn,"\n\n")
if (_time != None):
print(color.YELLOW+"< < < Data available for these time periods > > >\033[0m \n")
print(info.year)
if (_desc != None):
print(color.YELLOW+"< < < Here are the experiment descriptions > > >\033[0m")
for item in info.exp:
print("\n\n"+color.CYAN+str(item)+":\033[0m \n"+CMIP6DB._get_definition(item))
print(color.RED+"\n\n <===============Exiting now!================>\033[0m\n\n")
raise SystemExit
def DownloadCmip6(**kwargs):
global startTime
_var = kwargs.get('variable', None)
_mod = kwargs.get('model', None)
_exp = kwargs.get('experiment', None)
_freq = kwargs.get('frequency', None)
_realm = kwargs.get('realm', None)
_check = kwargs.get('check', None)
rlzn = kwargs.get('rlzn', None)
year = kwargs.get('year', None)
path = kwargs.get('path', None)
skip = kwargs.get('skip', None)
cr = kwargs.get('cr', None)
search = SearchDB()
if (_check == 'Yes') or (_check == 'yes'):
search._set_check('Yes')
else:
search._set_check('No')
if (_mod != None):
try:
search.model = _mod
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_mod) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_exp != None):
try:
search.experiment = _exp
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_exp) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_var != None):
try:
search.variable = _var
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_var) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_freq != None):
try:
search.frequency = _freq
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_freq) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_realm != None):
try:
search.realm = _realm
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_realm) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
print("\nFinding server . . .")
links = search.get_links(0)
if (links == []):
print('\n' + color.LRED + '<<Invalid search items!>>' + color.END)
print('\n' + color.UNDERLINE + color.BOLD + 'TIPS 1:' + color.END +
' Use the check (-c) argument to check your inputs.' + color.END)
print('\n' + color.UNDERLINE + color.BOLD + 'TIPS 2:' + color.END +
' Use CMIP6DB module to look for currently available '
'models/experiments/variables and so on . . .')
raise SystemExit
if (year != None):
info = search.get_info()
yr_links = []
if int(year) > 0:
end_year = int(info.year[0]) + int(year)
else:
end_year = int(info.year[len(info.year) - 1]) + int(year)
interested_years = []
for y in info.year:
if (int(year) > 0) and (int(y) - 1 < end_year):
interested_years.append(y)
elif (int(year) < 0) and (end_year < int(y) + 1):
interested_years.append(y)
else:
continue
for item in interested_years:
for link in links:
if '_' + item in link:
yr_links.append(link)
links = yr_links
if (skip != None):
links = _get_skipped_links(links, skip)
unused_links = []
if (rlzn != None):
all_rlzn = _realizations(links)._all_realizations()
new_links = _get_rlzn_links(rlzn, all_rlzn, links)
links = new_links
if (cr != None):
info = search.get_info()
exps = info.exp
ers = []
for i in range(len(exps)):
ers1 = [s for s in links if exps[i] in s]
r1 = _realizations(ers1)._all_realizations()
if i == 1:
ers2 = [s for s in links if exps[i - 1] in s]
r2 = _realizations(ers2)._all_realizations()
elif i > 1:
r2 = ers
else:
ers2 = ers1
r2 = _realizations(ers2)._all_realizations()
ers = list(set(r1) & set(r2))
ers = ['_r' + str(s) + 'i' for s in ers]
links = [s for s in links if any(xs in s for xs in ers)]
if (path == None):
dir_path = _dir_path()._make_dir()
else:
dir_path = Path(path)
if not os.path.isdir(dir_path):
print('creating ', dir_path)
os.makedirs(str(dir_path))
os.chdir(dir_path)
n = 0
m = 0
manual = 0
passed_urls = []
for url in links:
startTime = time.time()
try:
dl_cmip6(url, dir_path)
n = n + 1
passed_urls.append(url)
except TooSlowException:
print("Removing file . . .\n")
os.remove(url.split('/')[len(url.split('/')) - 1])
m = m + 1
startTime = time.time()
except KeyboardInterrupt:
print("\nInterrupted! Removing file . . .\n")
os.remove(url.split('/')[len(url.split('/')) - 1])
break
except urllib.error.HTTPError:
m = m + 1
manual = manual + 1
print("\n" + color.RED +
"<<401 Unauthorized: restricted access!!>>" + color.END +
"\n")
print(
color.UNDERLINE + "From ESGF:" + color.END +
" Before you can download this data, you have to join a data access control group \nsince acknowledgement of a policy is a condition for this data download."
)
print("\nRequires registration/manual download . . . :(")
pass
except:
m = m + 1
os.remove(url.split('/')[len(url.split('/')) - 1])
pass
print("\nFinished downloading.")
print("\n\nDownloaded ", n, " out of ", n + m, " files.")
if (m > 0):
print("\n\nRe-run the script for the missing files.")
if (manual > 0):
print("\n\n", manual, " files require an ESGF account/openID.")
print(
"\nwget script created for these files!\nUse it with your openid/password >> 'wget_script -H'"
)
with HidePrint():
original_links = search.get_links(1)
unused_links = list(set(original_links) - set(links))
_manual_wget(passed_urls, unused_links)
def SearchCmip6(**kwargs):
_var = kwargs.get('variable', None)
_mod = kwargs.get('model', None)
_exp = kwargs.get('experiment', None)
_freq = kwargs.get('frequency', None)
_realm = kwargs.get('realm', None)
_check = kwargs.get('check', None)
_desc = kwargs.get('desc', None)
module = kwargs.get('module', None)
if (module == 'on'):
_mod_help()
if (_var == 'show') or (_var == 'Show'):
ModDB = CMIP6DB()
print(ModDB.var_stdName())
if (_mod == 'show') or (_mod == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_models())
if (_exp == 'show') or (_exp == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_experiments())
if (_freq == 'show') or (_freq == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_frequencies())
if (_realm == 'show') or (_realm == 'Show'):
ModDB = CMIP6DB()
print(ModDB.available_realmns())
raise SystemExit
search = SearchDB()
if (_check == 'Yes') or (_check == 'yes'):
print(
'\n' + color.UNDERLINE + color.BOLD + 'TIPS:' + color.END +
" If you are not sure about what you are looking for use CMIP6DB module \n to look for currently available models/experiments/variables and so on . . ."
+ color.END)
search._set_check('Yes')
else:
print('\n' + color.UNDERLINE + color.BOLD + 'TIPS:' + color.END +
" Use the check (-c) argument to check your inputs." + color.END)
search._set_check('No')
if (_mod != None):
try:
search.model = _mod
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_mod) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_exp != None):
try:
search.experiment = _exp
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_exp) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_var != None):
try:
search.variable = _var
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_var) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_freq != None):
try:
search.frequency = _freq
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_freq) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
if (_realm != None):
try:
search.realm = _realm
except ValueError as ve:
print(color.LRED +
"\n<<No options available.>>\n\nPlease make sure " +
str(_realm) + " exists." + color.END)
print(ve)
except Exception as ee:
print('\nDid you mean any of the above?')
print(ee)
info = search.get_info()
print(
color.LGREEN +
"\n\n Currently available models based on your search: \n\n" +
color.END, info.mod)
print(
color.LGREEN +
"\nCurrently available variables based on your search: \n\n" +
color.END, info.var)
print(
color.LGREEN +
"\nCurrently available experiments based on your search: \n\n" +
color.END, info.exp, "\n\n")
print(color.LGREEN + "\nNumber of files:" + color.END, info.n_files,
"\n\n")
print(color.LGREEN + "\nAvailable realizations:" + color.END, info.rlzn,
"\n\n")
if (_desc != None):
print(color.YELLOW +
"< < < Here are the experiment descriptions > > >\033[0m")
for item in info.exp:
print("\n\n" + color.CYAN + str(item) + ":\033[0m \n" +
CMIP6DB._get_definition(item))
print(
color.RED +
"\n\n <===============Exiting now!================>\033[0m\n\n")
raise SystemExit